<jupyter_start><jupyter_text>Last Update<jupyter_code>!date<jupyter_output><empty_output><jupyter_text>Here is the code<jupyter_code>!apt-get install openjdk-8-jdk-headless -qq > /dev/null
!wget -q https://www-us.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz
!tar xf spark-2.4.4-bin-hadoop2.7.tgz
!pip install -q findspark<jupyter_output><empty_output><jupyter_text><jupyter_code>import os
os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-8-openjdk-amd64"
os.environ["SPARK_HOME"] = "/content/spark-2.4.4-bin-hadoop2.7"
import findspark
findspark.init()
from pyspark.sql import SparkSession
spark = SparkSession.builder.master("local[*]").getOrCreate()
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline

import seaborn as sns
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.evaluation import RegressionEvaluator
!wget https://www.dropbox.com/s/fhf4aba59ey0i2r/salary_data.csv
help(spark.read.csv)
df_dataset = spark.read.option("delimiter", "\t").csv(path='salary_data.csv', header = True, inferSchema = True)
df_dataset.show(6)
df_dataset.printSchema()
df = df_dataset.toPandas()
sns.scatterplot(x='YearsExperience', y='Salary', data=df)
data2 = df_dataset.select(df_dataset.YearsExperience, df_dataset.Salary.alias('label'))
data2.show(5)
assembler = VectorAssembler().setInputCols(['YearsExperience',])\
                            .setOutputCol('features')
vec_data = assembler.transform(data2)

vec_data = vec_data.select(['features', 'label'])
vec_data.show(5)
vec_data.printSchema()

train_data, test_data = vec_data.randomSplit([0.7,0.3])
train_data.show(5)
from pyspark.ml.regression import LinearRegression

lr = LinearRegression()
model = lr.fit(train_data)

print("Coefficients: " + str(model.coefficients))
print("Intercept: " + str(model.intercept))
print("")
print ("##","Mean squared error: % .6f" % model.summary.meanSquaredError)
print ("##","RMSE: % .6f" % model.summary.rootMeanSquaredError)
print ("##","R-squared: %f" %  model.summary.r2)
print ("##","Total iterations: %i" % model.summary.totalIterations)<jupyter_output>Coefficients: [9443.94990078018]
Intercept: 26139.66350205229

## Mean squared error:  40440538.539433
## RMSE:  6359.287581
## R-squared: 0.941949
## Total iterations: 1<jupyter_text>R-squared : 0.94 artinya tingkat akurasi sekitar 94%<jupyter_code>print("ObjectiveHistory: %s" % str(model.summary.objectiveHistory))
model.summary.residuals.show()
lr_predictions = model.transform(test_data)
lr_predictions.select("prediction","label","features").show(5)
lr_evaluator = RegressionEvaluator(predictionCol="prediction", \
                                   labelCol="label",metricName="r2")
print("R Squared (R2) on test data = %g" % lr_evaluator.evaluate(lr_predictions))
test_result = model.evaluate(test_data)
print("Root Mean Squared Error (RMSE) on test data = %g" % test_result.rootMeanSquaredError)<jupyter_output>Root Mean Squared Error (RMSE) on test data = 3637.48
{'fraction_non_alphanumeric': 0.14998314796090326, 'fraction_numerical': 0.03977081226828446, 'mean_word_length': 5.983529411764706, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 16, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '361735', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 1184, 'n_tokens_neox': 1092, 'n_words': 179, 'path': 'Berkenalan_Dengan_Linear_Regression_pada_PySpark_MLlib.ipynb', 'repo_name': 'project303/Ngulik-Big-Data-dan-AI', 'size': '2967'}