<jupyter_start><jupyter_text>Pipeline<jupyter_code>trainDF, testDF = spark_df.randomSplit([0.8, 0.2], seed=42)
print(trainDF.cache().count())
print(testDF.count())
display(trainDF.head(5))
from pyspark.ml.feature import VectorAssembler

assemblerInputs = ['crim', 'zn', 'indus', 'chas', 'nox', 'rm', 'age', 'dis', 'rad', 'tax','ptratio', 'b', 'lstat']
vecAssembler = VectorAssembler(inputCols=assemblerInputs, outputCol="features")
from pyspark.ml.regression import LinearRegression

lr = LinearRegression(featuresCol="features", labelCol="target", maxIter=10, regParam=0.3, elasticNetParam=0.8)
from pyspark.ml import Pipeline

pipeline = Pipeline(stages=[vecAssembler, lr])
pipelineModel = pipeline.fit(trainDF)

predDF = pipelineModel.transform(testDF)
display(predDF.select("features", "target", "prediction").show(5))<jupyter_output><empty_output><jupyter_text>No Pipeline<jupyter_code>from pyspark.ml.feature import VectorAssembler

assemblerInputs = ['crim', 'zn', 'indus', 'chas', 'nox', 'rm', 'age', 'dis', 'rad', 'tax','ptratio', 'b', 'lstat']
vectorAssembler = VectorAssembler(inputCols = assemblerInputs, outputCol = 'features')
spark_df_v = vectorAssembler.transform(spark_df)
display(spark_df_v.head(5))
trainDF, testDF = spark_df_v.randomSplit([0.8, 0.2], seed=42)
print(trainDF.cache().count())
print(testDF.count())
from pyspark.ml.regression import LinearRegression

lr = LinearRegression(featuresCol = 'features', labelCol='target', maxIter=10, regParam=0.3, elasticNetParam=0.8)
lr_model = lr.fit(trainDF)
print("Coefficients: " + str(lr_model.coefficients))
print("Intercept: " + str(lr_model.intercept))
trainingSummary = lr_model.summary
print("RMSE: %f" % trainingSummary.rootMeanSquaredError)
print("r2: %f" % trainingSummary.r2)
lr_predictions = lr_model.transform(testDF)
display(lr_predictions.select("target","features","prediction").show(5))
from pyspark.ml.evaluation import RegressionEvaluator

lr_evaluator = RegressionEvaluator(predictionCol="prediction", labelCol="target",metricName="r2")
print("R Squared (R2) on test data = %g" % lr_evaluator.evaluate(lr_predictions))
test_result = lr_model.evaluate(testDF)
print("Root Mean Squared Error (RMSE) on test data = %g" % test_result.rootMeanSquaredError)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15777875716174528, 'fraction_numerical': 0.014103129131776113, 'mean_word_length': 9.225225225225225, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '533299', 'license': 'MIT', 'n_tokens_mistral': 828, 'n_tokens_neox': 811, 'n_words': 139, 'path': 'ml/databricks_mllib_demo.ipynb', 'repo_name': 'grishenkovp/databricks', 'size': '2269'}