<jupyter_start><jupyter_text>reference: https://github.com/combust/mleap<jupyter_code>%AddDeps ml.combust.mleap mleap-spark_2.11 0.5.0 --transitive
%AddDeps com.databricks spark-avro_2.11 3.0.1
import org.apache.spark.sql.SparkSession
val spark = SparkSession.builder().master("local").getOrCreate()
// MLeap/Bundle.ML Serialization Libraries
import ml.combust.mleap.spark.SparkSupport._
import resource._
import ml.combust.bundle.BundleFile
import org.apache.spark.ml.bundle.SparkBundleContext
import ml.combust.bundle.BundleFile
import ml.combust.mleap.spark.SparkSupport._
import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.bundle.SparkBundleContext
import org.apache.spark.ml.feature.{Binarizer, StringIndexer}
import org.apache.spark.sql._
import org.apache.spark.sql.functions._
import resource._
import sys.process._
"curl https://raw.githubusercontent.com/combust/mleap/master/examples/spark-demo.csv -o /tmp/spark-demo.csv" !
"hadoop fs -mkdir -p /datasets" !
"hadoop fs -put /tmp/spark-demo.csv /datasets/spark-demo.csv" !
val datasetName = "/datasets/spark-demo.csv"
val dataframe: DataFrame = spark.sqlContext.read.format("csv")
    .option("header", true)
    .load(datasetName)
    .withColumn("test_double", col("test_double").cast("double"))
// User out-of-the-box Spark transformers like you normally would
  val stringIndexer = new StringIndexer().
    setInputCol("test_string").
    setOutputCol("test_index")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13528212100611828, 'fraction_numerical': 0.008157715839564922, 'mean_word_length': 5.158995815899582, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '443500', 'license': 'Apache-2.0', 'n_tokens_mistral': 524, 'n_tokens_neox': 498, 'n_words': 85, 'path': 'examples/Simple Spark ML pipeline Scala.ipynb', 'repo_name': 'connected-bsamadi/dataproc-tools', 'size': '1471'}