<jupyter_start><jupyter_text>Logistic Regression Project in Spark:    With titanic dataset<jupyter_code>from pyspark.sql import SparkSession
from pyspark.ml.classification import LogisticRegression
spark = SparkSession.builder.appName('log_reg_project').getOrCreate()
data = spark.read.csv('../datasets/titanic.csv',inferSchema=True,header=True)
data.show()
data.describe().show()<jupyter_output>+-------+-----------------+-------------------+------------------+--------------------+------+------------------+------------------+-------------------+------------------+-----------------+-----+--------+
|summary|      PassengerId|           Survived|            Pclass|                Name|   Sex|               Age|             SibSp|              Parch|            Ticket|             Fare|Cabin|Embarked|
+-------+-----------------+-------------------+------------------+--------------------+------+------------------+------------------+-------------------+------------------+-----------------+-----+--------+
|  count|              891|                891|               891|                 891|   891|               714|               891|                891|               891|              891|  204|     889|
|   mean|            446.0| 0.3838383838383838| 2.308641975308642|                null|  null| 29.69911764705882|0.5230078563411896|0.38159371492704824|260318.54916792738| 32.20420[...]<jupyter_text>Select the columns that we'll use in our model<jupyter_code>data.columns
cols = data.select('Pclass',
 'Survived',
 'Sex',
 'Age',
 'SibSp',
 'Parch',
 'Fare',
 'Embarked')<jupyter_output><empty_output><jupyter_text>Drop any null values<jupyter_code>final_data = cols.na.drop()
final_data.printSchema()<jupyter_output>root
 |-- Pclass: integer (nullable = true)
 |-- Survived: integer (nullable = true)
 |-- Sex: string (nullable = true)
 |-- Age: double (nullable = true)
 |-- SibSp: integer (nullable = true)
 |-- Parch: integer (nullable = true)
 |-- Fare: double (nullable = true)
 |-- Embarked: string (nullable = true)<jupyter_text>Convert categorical column to numerical<jupyter_code>from pyspark.ml.feature import (VectorAssembler,VectorIndexer,
                                OneHotEncoder,StringIndexer)
gender_indexer = StringIndexer(inputCol='Sex',outputCol='SexIndex')
# A B C
# 0 1 2
# ONE HOT ENCODER
# A = [1,0,0]
# B = [0,1,0] etc.
gender_encoder = OneHotEncoder(inputCol='SexIndex',outputCol='SexVec')
embark_indexer = StringIndexer(inputCol='Embarked',outputCol='EmbarkIndex')
embark_encoder = OneHotEncoder(inputCol='EmbarkIndex',outputCol='EmbarkVec')<jupyter_output><empty_output><jupyter_text>Assemble all columns to make a feature column<jupyter_code>assembler = VectorAssembler(inputCols=['Pclass','SexVec','Age','SibSp','Parch','Fare','EmbarkVec'],outputCol='features')
log_reg = LogisticRegression(featuresCol='features',labelCol='Survived')<jupyter_output><empty_output><jupyter_text>Creating the Pipeline<jupyter_code>from pyspark.ml.pipeline import Pipeline
pipeline = Pipeline(stages=[gender_indexer,embark_indexer,gender_encoder,
                            embark_encoder,assembler,log_reg])
# Train Test Split
train_data, test_data = final_data.randomSplit([0.7,0.3])
model = pipeline.fit(train_data)
result = model.transform(test_data)
result.select('Survived','prediction').show()
from pyspark.ml.evaluation import BinaryClassificationEvaluator
evaluator = BinaryClassificationEvaluator(rawPredictionCol='prediction',labelCol='Survived')
AUC = evaluator.evaluate(result)
AUC
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
acc_eval = MulticlassClassificationEvaluator(labelCol='Survived',metricName='accuracy')
print(acc_eval.evaluate(result))<jupyter_output>0.8476190476190476
{'fraction_non_alphanumeric': 0.21765804214457188, 'fraction_numerical': 0.047479327820752204, 'mean_word_length': 2.672869735553379, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '243660', 'license': 'MIT', 'n_tokens_mistral': 1262, 'n_tokens_neox': 1127, 'n_words': 182, 'path': 'Code/Day67-LogRegSparkProject.ipynb', 'repo_name': 'MDSADABWASIM/ML-Project', 'size': '3749'}