<jupyter_start><jupyter_text>Import Libraries<jupyter_code>from pyspark.sql import SparkSession
spark= SparkSession.builder.appName('Customers').getOrCreate()
from pyspark.ml.regression import LinearRegression

dataset=spark.read.csv("Ecommerce_Customers.csv",inferSchema=True,header=True)<jupyter_output><empty_output><jupyter_text>Perform EDA<jupyter_code>dataset
dataset.show()
dataset.printSchema()
sklearn

x1,X2,X3,X4,X5    Y1 ---->model-->prediction

[X1,X2,X3,X4,X5]  Y1---->model--->prediction
from pyspark.ml.linalg import Vectors
from pyspark.ml.feature import VectorAssembler
featureassembler=VectorAssembler(inputCols=["Avg Session Length","Time on App","Time on Website","Length of Membership"],outputCol="Independent Features")
output=featureassembler.transform(dataset)
output.show()
output.select("Independent Features").show()
output.columns
finalized_data=output.select("Independent Features","Yearly Amount Spent")
finalized_data.show()
## 
train_data,test_data=finalized_data.randomSplit([0.75,0.25])
regressor=LinearRegression(featuresCol='Independent Features', labelCol='Yearly Amount Spent')
regressor=regressor.fit(train_data)
regressor.coefficients
regressor.intercept
pred_results=regressor.evaluate(test_data)
pred_results.predictions.show(40)<jupyter_output>+--------------------+-------------------+------------------+
|Independent Features|Yearly Amount Spent|        prediction|
+--------------------+-------------------+------------------+
|[29.53242897,10.9...|        408.6403511| 397.7314622481499|
|[30.39318454,11.8...|        319.9288698| 331.4354817055371|
|[30.73772037,12.6...|        461.7807422|451.00378856214934|
|[30.83643267,13.1...|        467.5019004| 471.6315593383845|
|[31.04722214,11.1...|        392.4973992| 387.6961791855133|
|[31.06621816,11.7...|        448.9332932|461.71709673379814|
|[31.35847719,12.8...|        495.1759504|491.12482994053585|
|[31.38958548,10.9...|        410.0696111| 409.3081755667929|
|[31.42522688,13.2...|        530.7667187| 534.6343118465913|
|[31.5261979,12.04...|        409.0945262|417.94636309938346|
|[31.53160448,13.3...|        436.5156057| 432.7242184024992|
|[31.57020083,13.3...|        545.9454921| 563.5592281627285|
|[31.62536013,13.1...|        376.3369008|380.96043463635397|
|[31.654[...]
{'fraction_non_alphanumeric': 0.20322721325774096, 'fraction_numerical': 0.2341910161360663, 'mean_word_length': 3.9761388286334056, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '135407', 'license': 'MIT', 'n_tokens_mistral': 1184, 'n_tokens_neox': 858, 'n_words': 100, 'path': 'final-project/.ipynb_checkpoints/Pyspark Linear Regression-checkpoint.ipynb', 'repo_name': 'Andre-Williams22/Data-Science-In-Production', 'size': '2293'}