<jupyter_start><jupyter_text>The other 3 are casual and registered and we are not going to bother about them.<jupyter_code>case class BikeRentals(season: Int, yr: Int, mnth: Int, hr: Int, holiday: Int, weekday: Int, workingday: Int, weathersit: Int, temp: Double, atemp: Double, hum: Double, windspeed: Double, cnt: Double)
def parseString(recs: String): BikeRentals = {
    val fields = recs.split(",")
    assert(fields.size==17)
    BikeRentals(fields(2).toInt, fields(3).toInt, fields(4).toInt, fields(5).toInt, fields(6).toInt, fields(7).toInt, fields(8).toInt, fields(9).toInt, fields(10).toDouble, fields(11).toDouble, fields(12).toDouble, fields(13).toDouble, fields(16).toDouble)
}
val data = records.map(parseString).toDF()
data.show(3)
val featureCols = Array("season", "yr", "mnth", "hr", "holiday", "weekday", "workingday", "weathersit", "temp", "atemp", "hum", "windspeed", "cnt")
val assembler = new VectorAssembler().setInputCols(featureCols).setOutputCol("features")
val dataDF = assembler.transform(data)<jupyter_output><empty_output><jupyter_text>We need to convert the output column to label so that the algorithms can find it.<jupyter_code>val dataDF1 = dataDF.withColumnRenamed("cnt", "label")
dataDF1.show(3)
val Array(train, test) = dataDF1.randomSplit(Array(0.8, 0.2))
println(train.count, rawData.count)
train.show(3)<jupyter_output>+------+---+----+---+-------+-------+----------+----------+----+------+----+---------+-----+--------------------+
|season| yr|mnth| hr|holiday|weekday|workingday|weathersit|temp| atemp| hum|windspeed|label|            features|
+------+---+----+---+-------+-------+----------+----------+----+------+----+---------+-----+--------------------+
|     1|  0|   1|  0|      0|      0|         0|         1|0.04|0.0758|0.57|   0.1045| 22.0|[1.0,0.0,1.0,0.0,...|
|     1|  0|   1|  0|      0|      0|         0|         1| 0.1|0.0758|0.42|   0.3881| 25.0|[1.0,0.0,1.0,0.0,...|
|     1|  0|   1|  0|      0|      0|         0|         1|0.16|0.1818| 0.8|   0.1045| 33.0|[1.0,0.0,1.0,0.0,...|
+------+---+----+---+-------+-------+----------+----------+----+------+----+---------+-----+--------------------+
only showing top 3 rows<jupyter_text>Model1: Linear Regression<jupyter_code>val lr = new LinearRegression()
    .setMaxIter(1000)
    .setRegParam(0.3)
    .setElasticNetParam(0.8)
//Fit the model
val lrModel = lr.fit(train)
// Print the coefficients and intercept for linear regression
println(s"Coefficients: ${lrModel.coefficients} Intercept: ${lrModel.intercept}")
// Summarize the model over the training set and print out some metrics
val trainingSummary = lrModel.summary
println(s"numIterations: ${trainingSummary.totalIterations}")
println(s"objectiveHistory: [${trainingSummary.objectiveHistory.mkString(",")}]")
trainingSummary.residuals.show()
println(s"RMSE: ${trainingSummary.rootMeanSquaredError}")
println(s"r2: ${trainingSummary.r2}")
val predictions = lrModel.transform(test)
predictions.show(3)
val eval = new RegressionEvaluator().setMetricName("rmse").setLabelCol("label").setPredictionCol("prediction")
val rmse = eval.evaluate(predictions)<jupyter_output><empty_output><jupyter_text>Model 2: generalised linear regression.<jupyter_code>import org.apache.spark.ml.regression.GeneralizedLinearRegression
val glr = new GeneralizedLinearRegression()
  .setFamily("gaussian")
  .setLink("identity")
  .setMaxIter(1000)
  .setRegParam(0.3)
// Fit the model
val model = glr.fit(train)
// Print the coefficients and intercept for generalized linear regression model
println(s"Coefficients: ${model.coefficients}")
println(s"Intercept: ${model.intercept}")

// Summarize the model over the training set and print out some metrics
val summary = model.summary
println(s"Coefficient Standard Errors: ${summary.coefficientStandardErrors.mkString(",")}")
println(s"T Values: ${summary.tValues.mkString(",")}")
println(s"P Values: ${summary.pValues.mkString(",")}")
println(s"Dispersion: ${summary.dispersion}")
println(s"Null Deviance: ${summary.nullDeviance}")
println(s"Residual Degree Of Freedom Null: ${summary.residualDegreeOfFreedomNull}")
println(s"Deviance: ${summary.deviance}")
println(s"Residual Degree Of Freedom: ${summary.residualDegreeOfFreedom}")
println(s"AIC: ${summary.aic}")
println("Deviance Residuals: ")
summary.residuals().show()
val predictions = model.transform(test)
predictions.show(3)
val eval = new RegressionEvaluator().setMetricName("rmse").setLabelCol("label").setPredictionCol("prediction")
val rmse = eval.evaluate(predictions)<jupyter_output><empty_output><jupyter_text>Decision tree regression<jupyter_code>import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.evaluation.RegressionEvaluator
import org.apache.spark.ml.feature.VectorIndexer
import org.apache.spark.ml.regression.DecisionTreeRegressionModel
import org.apache.spark.ml.regression.DecisionTreeRegressor
// Automatically identify categorical features, and index them.
// Here, we treat features with > 4 distinct values as continuous.
val featureIndexer = new VectorIndexer()
  .setInputCol("features")
  .setOutputCol("indexedFeatures")
  .setMaxCategories(4)
  .fit(dataDF1)

// Train a DecisionTree model.
val dt = new DecisionTreeRegressor()
  .setLabelCol("label")
  .setFeaturesCol("indexedFeatures")

// Chain indexer and tree in a Pipeline.
val pipeline = new Pipeline()
  .setStages(Array(featureIndexer, dt))

// Train model. This also runs the indexer.
val model = pipeline.fit(train)

// Make predictions.
val predictions = model.transform(test)

// Select example rows to display.
predictions.select("prediction", "label", "features").show(5)

// Select (prediction, true label) and compute test error.
val evaluator = new RegressionEvaluator()
  .setLabelCol("label")
  .setPredictionCol("prediction")
  .setMetricName("rmse")
val rmse = evaluator.evaluate(predictions)
println("Root Mean Squared Error (RMSE) on test data = " + rmse)

val treeModel = model.stages(1).asInstanceOf[DecisionTreeRegressionModel]
println("Learned regression tree model:\n" + treeModel.toDebugString)<jupyter_output>+------------------+-----+--------------------+
|        prediction|label|            features|
+------------------+-----+--------------------+
|35.831381733021075| 39.0|(13,[0,2,7,8,9,10...|
| 4.497063142437592|  7.0|[1.0,0.0,1.0,0.0,...|
| 4.497063142437592|  5.0|[1.0,0.0,1.0,0.0,...|
| 12.62093023255814| 12.0|[1.0,0.0,1.0,0.0,...|
| 4.497063142437592|  5.0|[1.0,0.0,1.0,0.0,...|
+------------------+-----+--------------------+
only showing top 5 rows

Root Mean Squared Error (RMSE) on test data = 15.233787588849488
Learned regression tree model:
DecisionTreeRegressionModel (uid=dtr_cc3e9578dd9a) of depth 5 with 63 nodes
  If (feature 12 <= 259.5)
   If (feature 12 <= 102.5)
    If (feature 12 <= 40.5)
     If (feature 12 <= 17.5)
      If (feature 12 <= 8.5)
       Predict: 4.497063142437592
      Else (feature 12 > 8.5)
       Predict: 12.62093023255814
     Else (feature 12 > 17.5)
      If (feature 12 <= 31.5)
       Predict: 24.29988465974625
      Else (feature 12 > 31.5)
       [...]<jupyter_text>Random forest regression<jupyter_code>import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.evaluation.RegressionEvaluator
import org.apache.spark.ml.feature.VectorIndexer
import org.apache.spark.ml.regression.{RandomForestRegressionModel, RandomForestRegressor}

// Automatically identify categorical features, and index them.
// Set maxCategories so features with > 4 distinct values are treated as continuous.
val featureIndexer = new VectorIndexer()
  .setInputCol("features")
  .setOutputCol("indexedFeatures")
  .setMaxCategories(4)
  .fit(dataDF1)

// Train a RandomForest model.
val rf = new RandomForestRegressor()
  .setLabelCol("label")
  .setFeaturesCol("indexedFeatures")

// Chain indexer and forest in a Pipeline.
val pipeline = new Pipeline()
  .setStages(Array(featureIndexer, rf))

// Train model. This also runs the indexer.
val model = pipeline.fit(train)

// Make predictions.
val predictions = model.transform(test)

// Select example rows to display.
predictions.select("prediction", "label", "features").show(5)

// Select (prediction, true label) and compute test error.
val evaluator = new RegressionEvaluator()
  .setLabelCol("label")
  .setPredictionCol("prediction")
  .setMetricName("rmse")
val rmse = evaluator.evaluate(predictions)
println("Root Mean Squared Error (RMSE) on test data = " + rmse)

val rfModel = model.stages(1).asInstanceOf[RandomForestRegressionModel]
println("Learned regression forest model:\n" + rfModel.toDebugString)<jupyter_output>+------------------+-----+--------------------+
|        prediction|label|            features|
+------------------+-----+--------------------+
|30.205306987378343| 39.0|(13,[0,2,7,8,9,10...|
| 27.05189758183248|  7.0|[1.0,0.0,1.0,0.0,...|
|22.462892752778828|  5.0|[1.0,0.0,1.0,0.0,...|
|21.096706702842148| 12.0|[1.0,0.0,1.0,0.0,...|
|20.689009715084815|  5.0|[1.0,0.0,1.0,0.0,...|
+------------------+-----+--------------------+
only showing top 5 rows

Root Mean Squared Error (RMSE) on test data = 34.07138081037501
Learned regression forest model:
RandomForestRegressionModel (uid=rfr_27251a44ebe7) with 20 trees
  Tree 0 (weight 1.0):
    If (feature 12 <= 278.5)
     If (feature 12 <= 113.5)
      If (feature 12 <= 51.5)
       If (feature 12 <= 23.5)
        If (feature 6 in {1.0})
         Predict: 8.654848800834202
        Else (feature 6 not in {1.0})
         Predict: 11.522875816993464
       Else (feature 12 > 23.5)
        If (feature 10 <= 0.415)
         Predict: 39.640776699[...]<jupyter_text>Gradient-boosted tree regression<jupyter_code>import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.evaluation.RegressionEvaluator
import org.apache.spark.ml.feature.VectorIndexer
import org.apache.spark.ml.regression.{GBTRegressionModel, GBTRegressor}

// Automatically identify categorical features, and index them.
// Set maxCategories so features with > 4 distinct values are treated as continuous.
val featureIndexer = new VectorIndexer()
  .setInputCol("features")
  .setOutputCol("indexedFeatures")
  .setMaxCategories(4)
  .fit(dataDF1)

// Split the data into training and test sets (30% held out for testing).
val Array(trainingData, testData) = dataDF1.randomSplit(Array(0.7, 0.3))

// Train a GBT model.
val gbt = new GBTRegressor()
  .setLabelCol("label")
  .setFeaturesCol("indexedFeatures")
  .setMaxIter(10)

// Chain indexer and GBT in a Pipeline.
val pipeline = new Pipeline()
  .setStages(Array(featureIndexer, gbt))

// Train model. This also runs the indexer.
val model = pipeline.fit(trainingData)

// Make predictions.
val predictions = model.transform(testData)

// Select example rows to display.
predictions.select("prediction", "label", "features").show(5)

// Select (prediction, true label) and compute test error.
val evaluator = new RegressionEvaluator()
  .setLabelCol("label")
  .setPredictionCol("prediction")
  .setMetricName("rmse")
val rmse = evaluator.evaluate(predictions)
println("Root Mean Squared Error (RMSE) on test data = " + rmse)

val gbtModel = model.stages(1).asInstanceOf[GBTRegressionModel]
println("Learned regression GBT model:\n" + gbtModel.toDebugString)<jupyter_output>+------------------+-----+--------------------+
|        prediction|label|            features|
+------------------+-----+--------------------+
| 24.21518454819239| 22.0|[1.0,0.0,1.0,0.0,...|
|  4.14947920710293|  5.0|[1.0,0.0,1.0,0.0,...|
|  4.14947920710293|  7.0|[1.0,0.0,1.0,0.0,...|
|3.0571537907650703|  3.0|[1.0,0.0,1.0,0.0,...|
| 11.89254762432865| 13.0|[1.0,0.0,1.0,0.0,...|
+------------------+-----+--------------------+
only showing top 5 rows

Root Mean Squared Error (RMSE) on test data = 13.370106514281224
Learned regression GBT model:
GBTRegressionModel (uid=gbtr_70903e1d529b) with 10 trees
  Tree 0 (weight 1.0):
    If (feature 12 <= 258.5)
     If (feature 12 <= 102.5)
      If (feature 12 <= 41.5)
       If (feature 12 <= 17.5)
        If (feature 12 <= 7.5)
         Predict: 4.134831460674158
        Else (feature 12 > 7.5)
         Predict: 11.877899877899878
       Else (feature 12 > 17.5)
        If (feature 12 <= 32.5)
         Predict: 24.824516129032258
        Els[...]<jupyter_text>Survival regression<jupyter_code>import org.apache.spark.ml.linalg.Vectors
import org.apache.spark.ml.regression.AFTSurvivalRegression

val quantileProbabilities = Array(0.3, 0.6)
val aft = new AFTSurvivalRegression()
  .setQuantileProbabilities(quantileProbabilities)
  .setQuantilesCol("quantiles")

val model = aft.fit(train)

// Print the coefficients, intercept and scale parameter for AFT survival regression
println(s"Coefficients: ${model.coefficients}")
println(s"Intercept: ${model.intercept}")
println(s"Scale: ${model.scale}")
model.transform(train).show(false)<jupyter_output><empty_output><jupyter_text>Isotonic Regression<jupyter_code>import org.apache.spark.ml.regression.IsotonicRegression

// Trains an isotonic regression model.
val ir = new IsotonicRegression()
val model = ir.fit(train)

println(s"Boundaries in increasing order: ${model.boundaries}\n")
println(s"Predictions associated with the boundaries: ${model.predictions}\n")

// Makes predictions.
model.transform(test).show()<jupyter_output>Boundaries in increasing order: [1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0,3.0,3.0,3.0,3.0,3.0,3.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0[...]
{'fraction_non_alphanumeric': 0.21123312583059384, 'fraction_numerical': 0.09960131496118067, 'mean_word_length': 3.497640767536961, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '162887', 'license': 'Apache-2.0', 'n_tokens_mistral': 6065, 'n_tokens_neox': 5501, 'n_words': 991, 'path': 'notebooks/linear regression multiple models.ipynb', 'repo_name': 'infinite-Joy/spark-scala-tutorial', 'size': '14297'}