<jupyter_start><jupyter_text>Linear Regression In Apache Spark<jupyter_code>from pyspark.sql import SparkSession
spark=SparkSession.builder.appName("ML Regression").getOrCreate()
from pyspark.ml.regression import   LinearRegression
training=spark.read.format('libsvm').load("C:/Users/User/Desktop/SparkFolder/Data/libsvmdata.txt")
training.show()
#Instantiate an Object out of the class
lr=LinearRegression(featuresCol="features",labelCol="label",predictionCol="prediction")
lrmodel=lr.fit(training)
lrmodel.coefficients
lrmodel.intercept
training_summary=lrmodel.summary
training_summary.meanAbsoluteError
training_summary.rootMeanSquaredError
alldf=spark.read.format('libsvm').load("C:/Users/User/Desktop/SparkFolder/Data/libsvmdata.txt")<jupyter_output><empty_output><jupyter_text>Train test Split Data<jupyter_code># Leversge tuple Unpacking
train_data,test_data=alldf.randomSplit([0.7,0.3])
train_data.count()
test_data.count()
lrmodel2=lr.fit(train_data)
lrmodel2.intercept
lrmodel2.coefficients<jupyter_output><empty_output><jupyter_text>Evaluate Test Data<jupyter_code>testresults=lrmodel2.evaluate(test_data)
testresults.predictions
testresults.predictionCol
testresults.meanSquaredError
testresults.r2
testresults.r2adj
testresults.r2
testresults.rootMeanSquaredError<jupyter_output><empty_output><jupyter_text>Predicting Data<jupyter_code>test_features=test_data.select("features")
test_features.show()<jupyter_output>+--------------------+
|            features|
+--------------------+
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
|(10,[0,1,2,3,4,5,...|
+--------------------+
only showing top 20 rows<jupyter_text>Use transform method to show predictions for each feature in the Dataframe<jupyter_code>predictions=lrmodel2.transform(test_features)  # Remember we use transform to show predictions. "Evaluate is for evaluation"
predictions.show()<jupyter_output>+--------------------+--------------------+
|            features|          prediction|
+--------------------+--------------------+
|(10,[0,1,2,3,4,5,...|-0.01797049724030...|
|(10,[0,1,2,3,4,5,...| -0.5975910018045235|
|(10,[0,1,2,3,4,5,...| -3.8320555367854676|
|(10,[0,1,2,3,4,5,...|   3.868499703478318|
|(10,[0,1,2,3,4,5,...|  1.9183434938055757|
|(10,[0,1,2,3,4,5,...|  1.4520960330708568|
|(10,[0,1,2,3,4,5,...|-0.26876430434246057|
|(10,[0,1,2,3,4,5,...|  0.6276947535694103|
|(10,[0,1,2,3,4,5,...|  2.9268400416900917|
|(10,[0,1,2,3,4,5,...| -0.8332530621271114|
|(10,[0,1,2,3,4,5,...|  0.3899205696172794|
|(10,[0,1,2,3,4,5,...| -0.6069598047590702|
|(10,[0,1,2,3,4,5,...|   4.449435089955961|
|(10,[0,1,2,3,4,5,...|  1.1720505359435194|
|(10,[0,1,2,3,4,5,...|  0.6130932672439465|
|(10,[0,1,2,3,4,5,...|  0.2841068569800861|
|(10,[0,1,2,3,4,5,...|  -1.941549012087568|
|(10,[0,1,2,3,4,5,...|   2.135503427562222|
|(10,[0,1,2,3,4,5,...|  1.8850825527828428|
|(10,[0,1,2,3,4,5,...|   3.93308[...]
{'fraction_non_alphanumeric': 0.2885967594007949, 'fraction_numerical': 0.20116172424335066, 'mean_word_length': 4.970802919708029, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '165956', 'license': 'Apache-2.0', 'n_tokens_mistral': 1849, 'n_tokens_neox': 1534, 'n_words': 143, 'path': '.ipynb_checkpoints/08.Linear Regression-checkpoint.ipynb', 'repo_name': 'kwabena55/Spark-for-Big-Data-and-Predictive-Analytics', 'size': '3271'}