<jupyter_start><jupyter_text>Examples Of Pyspark ML<jupyter_code>import pyspark
from pyspark.sql import SparkSession
spark = SparkSession.builder.appName('ml').getOrCreate()<jupyter_output><empty_output><jupyter_text>Read the Dataset<jupyter_code>tips = spark.read.csv('tips.csv', header=True, inferSchema = True)
tips.show()
tips.printSchema()
tips.columns<jupyter_output><empty_output><jupyter_text>Linear Regression  Handling categorical features<jupyter_code># Handling categorical features
from pyspark.ml.feature import StringIndexer
indexer = StringIndexer(inputCol = "sex", outputCol='sex_indexed')
df = indexer.fit(tips).transform(tips)
df.show()
#Categorical features convert to indexed
indexer = StringIndexer(inputCols = ["smoker","day","time"], outputCols =["smoker_indexed","day_indexed","time_indexed"])
df = indexer.fit(df).transform(df)
df.show()
df.columns
#Create independent features using vector assembler
from pyspark.ml.feature import VectorAssembler

featureassembler = VectorAssembler(inputCols=['tip','size','sex_indexed', 'smoker_indexed','day_indexed', 'time_indexed'],
                                   outputCol= "Independent Features")
output = featureassembler.transform(df)
output.show()
output.select('Independent Features').show()
finalized_data = output.select("Independent Features", "total_bill")
finalized_data.show()
#import Linearregression package
from pyspark.ml.regression import LinearRegression
#train test split
train_data, test_data = finalized_data.randomSplit([0.75,0.25])

#Create regression model
regressor = LinearRegression(featuresCol = 'Independent Features', labelCol='total_bill')
regressor = regressor.fit(train_data)
regressor.coefficients
regressor.intercept
# Create Prediction
pred_result = regressor.evaluate(test_data)
pred_result.predictions.show()
print('Linear Regression Model R Squared(R2) on test data: ', pred_result.r2)
print('Linear Regression Model Mean Absolute Error(MAE) on test data: ', pred_result.meanAbsoluteError)
print('Linear Regression Model Mean Squared Error(MSE) on test data: ', pred_result.meanSquaredError)<jupyter_output>Linear Regression Model Mean Squared Error(MSE) on test data:  34.04468655177479
{'fraction_non_alphanumeric': 0.11070615034168566, 'fraction_numerical': 0.010933940774487472, 'mean_word_length': 7.819277108433735, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '226928', 'license': 'Apache-2.0', 'n_tokens_mistral': 725, 'n_tokens_neox': 681, 'n_words': 147, 'path': 'PySpark_3/pyspark_notes_3.ipynb', 'repo_name': 'SeydaOzdemirr/Spark_intro', 'size': '2195'}