<jupyter_start><jupyter_text>Data Exploration<jupyter_code>df.printSchema()
# data set is imbalanced!
pos_cnt = df.filter(df['Churn'] == 1).count()
neg_cnt = df.filter(df['Churn'] == 0).count()
print('pos: ', pos_cnt, 'neg: ', neg_cnt)
for col, item in zip(df.columns, df.take(1)[0]):
    print(col + ': ' + str(item))<jupyter_output>Names: <NAME>
Age: 42.0
Total_Purchase: 11066.8
Account_Manager: 0
Years: 7.22
Num_Sites: 8.0
Onboard_date: 2013-08-30 07:00:40
Location: 10265 Elizabeth Mission Barkerburgh, AK 89518
Company: Harvey LLC
Churn: 1<jupyter_text>Data Cleansing<jupyter_code>my_cols = df.select('Age', 'Total_Purchase', 'Years', 'Num_Sites', 'Churn')
without_na = my_cols.na.drop()
without_na.count()
# no missing data was removed<jupyter_output><empty_output><jupyter_text>Create Feature Vector<jupyter_code>from pyspark.ml.feature import (VectorAssembler, VectorIndexer)
assembler = VectorAssembler(inputCols=['Age', 'Total_Purchase', 'Years', 'Num_Sites'], outputCol='features')
final_data = assembler.transform(my_cols)<jupyter_output><empty_output><jupyter_text>Train / Test Sets<jupyter_code>train, test = final_data.randomSplit([0.7, 0.3])<jupyter_output><empty_output><jupyter_text>Train Logistic Regression<jupyter_code>from pyspark.ml.classification import LogisticRegression
log_reg = LogisticRegression(featuresCol='features', labelCol='Churn')
fit_log_reg = log_reg.fit(train)<jupyter_output><empty_output><jupyter_text>Evaluate Logistic Regression<jupyter_code>from pyspark.ml.evaluation import BinaryClassificationEvaluator
evaluator = BinaryClassificationEvaluator(labelCol='Churn')
pred_labels = fit_log_reg.evaluate(test)
pred_labels.predictions.show()
results = evaluator.evaluate(pred_labels.predictions)
results<jupyter_output><empty_output><jupyter_text>Evaluate on New Data<jupyter_code>final_lr_model = log_reg.fit(final_data)
new_customers = spark.read.csv('../data/new_customers.csv',inferSchema=True, header=True)
new_customers.printSchema()
new_customers_df = assembler.transform(new_customers)
new_customers_df.take(1)
final_results = final_lr_model.transform(new_customers_df)
from pyspark.sql.functions import round
final_results.select('Names', 'Age', 'prediction', 'probability').show()<jupyter_output>+--------------+----+----------+--------------------+
|         Names| Age|prediction|         probability|
+--------------+----+----------+--------------------+
| <NAME>|37.0|       0.0|[0.91821394450559...|
|<NAME>|23.0|       1.0|[0.00249574827398...|
|  <NAME>|65.0|       1.0|[0.03129415226295...|
|<NAME>|32.0|       1.0|[0.00475889563977...|
|  <NAME>|32.0|       0.0|[0.78319303188952...|
| <NAME>|22.0|       1.0|[0.19177446955641...|
+--------------+----+----------+--------------------+
{'fraction_non_alphanumeric': 0.19781420765027322, 'fraction_numerical': 0.06156648451730419, 'mean_word_length': 4.918103448275862, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '60882', 'license': 'MIT', 'n_tokens_mistral': 1110, 'n_tokens_neox': 997, 'n_words': 142, 'path': 'apache-spark/python/logistic-regression/predicting-customer-churn.ipynb', 'repo_name': 'muhdlaziem/notes', 'size': '2797'}