<jupyter_start><jupyter_text>Check correlations of all variables wrt output variable<jupyter_code>from pyspark.sql.functions import corr
for col in df.columns[:-1]:
    df.select(corr(df[col], df['output'])).show()<jupyter_output>+-------------------+
|corr(var_1, output)|
+-------------------+
| 0.9187399607627283|
+-------------------+

+-------------------+
|corr(var_2, output)|
+-------------------+
|0.43652698913681093|
+-------------------+

+-------------------+
|corr(var_3, output)|
+-------------------+
| 0.4014958408311139|
+-------------------+

+-------------------+
|corr(var_4, output)|
+-------------------+
| 0.7909100204842113|
+-------------------+

+-------------------+
|corr(var_5, output)|
+-------------------+
| 0.7904806260381185|
+-------------------+<jupyter_text>Feature Engineering<jupyter_code>from pyspark.ml.linalg import Vector
from pyspark.ml.feature import VectorAssembler
df.columns
vec_asmblr = VectorAssembler(inputCols=df.columns[:-1], outputCol='features')
df = vec_asmblr.transform(df)
df = df.select('features','output')
df.printSchema()<jupyter_output>root
 |-- features: vector (nullable = true)
 |-- output: double (nullable = true)<jupyter_text>Splitting dataset<jupyter_code>train_df, test_df = df.randomSplit([0.75,0.25])
train_df.count(),test_df.count()<jupyter_output><empty_output><jupyter_text>Train model<jupyter_code>from pyspark.ml.regression import LinearRegression
lin_reg = LinearRegression(labelCol='output')
lin_reg = lin_reg.fit(train_df)
lin_reg.coefficients
lin_reg.intercept
test_predictions = lin_reg.evaluate(test_df)
test_predictions.r2
test_predictions.meanSquaredError<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.29468021518230725, 'fraction_numerical': 0.05977286312014345, 'mean_word_length': 2.8306636155606406, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '108481', 'license': 'MIT', 'n_tokens_mistral': 627, 'n_tokens_neox': 561, 'n_words': 76, 'path': 'Machine Learning with PySpark/Chapter 4 Linear Regression.ipynb', 'repo_name': 'sonwanesuresh95/Books-to-notebooks', 'size': '1673'}