<jupyter_start><jupyter_text>Feature Engineering<jupyter_code>total_songs = df.select('userId', 'song').groupBy('userId').count().withColumnRenamed('count', 'total_songs')
total_length = df.select('userId', 'length').groupBy('userId').sum().withColumnRenamed('sum(length)', 'total_length')
thumbs_up = df.select('userId', 'page').where(df.page == 'Thumbs Up').groupBy('userId').count().withColumnRenamed('count', 'thumbs_up')
thumbs_down = df.select('userId', 'page').where(df.page == 'Thumbs Down').groupBy('userId').count().withColumnRenamed('count', 'thumbs_down')
user_lifetime = df.select('userId', 'registration', 'ts').withColumn('tot_lifetime', (df.ts - df.registration)).groupBy('userId').agg({'tot_lifetime' : 'max'})\
    .withColumnRenamed('max(tot_lifetime)', 'tot_lifetime').select('userId', (col('tot_lifetime')/1000/3600/24).alias('user_lifetime'))
friends_added = df.select('userId', 'page').where(df.page == 'Add Friend').groupBy('userId').count().withColumnRenamed('count', 'friends_added')
songs_per_session = df.where('page == "NextSong"').groupby(['userId', 'sessionId']).count().groupby('userId')\
          .agg({'count' : 'avg'}).withColumnRenamed('avg(count)', 'songs_per_session')
total_artists = df.filter(df.page == "NextSong").select("userId", "artist").dropDuplicates().groupby("userId").count()\
         .withColumnRenamed("count", "total_artists")
gender = df.select('userId', 'gender').dropDuplicates().replace(['F', 'M'], ['1', '0'], 'gender').select('userId', col('gender').cast('int'))
playlist_added = df.select('userID','page').where(df.page == 'Add to Playlist').groupBy('userID').count().withColumnRenamed('count', 'playlist_added')
downgrade = df.select('userId', 'downgrade').dropDuplicates().select('userId', col('downgrade'))
user_churn = df.select('userId', col('user_churn')).dropDuplicates()

final_df = total_songs.join(total_length, 'userId', 'inner').join(thumbs_up, 'userId', 'inner')\
                .join(thumbs_down, 'userId', 'inner').join(user_lifetime, 'userId', 'inner').join(songs_per_session, 'userId', 'inner')\
                .join(friends_added, 'userId', 'inner').join(playlist_added, 'userId', 'inner').join(gender, 'userId', 'inner')\
                .join(total_artists, 'userId', 'inner').join(downgrade, 'userId', 'inner')\
                .join(user_churn, 'userId', 'inner').drop('userId')

final_df.show(10)<jupyter_output>+-----------+------------------+---------+-----------+------------------+------------------+-------------+--------------+------+-------------+---------+----------+
|total_songs|      total_length|thumbs_up|thumbs_down|     user_lifetime| songs_per_session|friends_added|playlist_added|gender|total_artists|downgrade|user_churn|
+-----------+------------------+---------+-----------+------------------+------------------+-------------+--------------+------+-------------+---------+----------+
|        137|25870.950739999997|        4|          3|14.328449074074074|              48.0|            3|             1|     1|           94|        0|         1|
|        395| 78695.82683999998|       15|          5| 53.32314814814814|              62.0|            2|             6|     0|          287|        0|         1|
|         84|15813.771070000003|        3|          1|105.42211805555554|20.666666666666668|            3|             2|     0|           61|        0|         0|
|       2156|   [...]<jupyter_text>Modeling<jupyter_code>assmblr = VectorAssembler(inputCols = ['total_songs', 'total_length', 'thumbs_up', 'thumbs_down', 'user_lifetime', 
                                       'songs_per_session', 'friends_added', 'playlist_added', 'gender', 'total_artists', 
                                       'downgrade'],
                          outputCol = "VectFeatures")
final_df = assmblr.transform(final_df)
scaler = StandardScaler(inputCol = 'VectFeatures', outputCol = "scaled_features", withStd = True)
scaler_model = scaler.fit(final_df)
final_df = scaler_model.transform(final_df)
data = final_df.select(final_df.user_churn.alias("label"), final_df.scaled_features.alias("features"))
train, rest = data.randomSplit([0.7, 0.3])
val, test = rest.randomSplit([0.5, 0.5])
train.select('label').agg({'label': 'sum'}).collect()
train.count()
dummy_classifier = val.withColumn('prediction', lit(0.0))

print('Dummy Classifier metrics:')
evaluator = MulticlassClassificationEvaluator(predictionCol='prediction')
print('Accuracy: {}'.format(evaluator.evaluate(dummy_classifier, {evaluator.metricName: "accuracy"})))
print('F-1 Score:{}'.format(evaluator.evaluate(dummy_classifier, {evaluator.metricName: "f1"})))
print('Recall Score:{}'.format(evaluator.evaluate(dummy_classifier, {evaluator.metricName: "weightedRecall"})))
lr = LogisticRegression(maxIter=15)
rf = RandomForestClassifier()
svc = LinearSVC(maxIter=15)
gb = GBTClassifier(maxIter=15)
model_lr = lr.fit(train)
results_lr = model_lr.transform(val)

print('Logistic Regression metrics:')
evaluator = MulticlassClassificationEvaluator(predictionCol='prediction')
print('Accuracy: {}'.format(evaluator.evaluate(results_lr, {evaluator.metricName: "accuracy"})))
print('F-1 Score:{}'.format(evaluator.evaluate(results_lr, {evaluator.metricName: "f1"})))
print('Recall Score:{}'.format(evaluator.evaluate(results_lr, {evaluator.metricName: "weightedRecall"})))
f1_evaluation = MulticlassClassificationEvaluator(metricName = 'f1')
param_grid = ParamGridBuilder().build()

lrs = LogisticRegression(maxIter=15)
cv_lr = CrossValidator(estimator = lrs, estimatorParamMaps = param_grid, evaluator = f1_evaluation, numFolds = 3)
model_lr_cv = cv_lr.fit(train)
results_lr_cv = model_lr_cv.transform(val)

print('Logistic Regression Cross-Val metrics:')
evaluator = MulticlassClassificationEvaluator(predictionCol='prediction')
print('Accuracy: {}'.format(evaluator.evaluate(results_lr_cv, {evaluator.metricName: "accuracy"})))
print('F-1 Score:{}'.format(evaluator.evaluate(results_lr_cv, {evaluator.metricName: "f1"})))
f1_evaluation = MulticlassClassificationEvaluator(metricName = 'weightedRecall')
param_grid = ParamGridBuilder().build()

lrs = LogisticRegression(maxIter=15)
cv_lr = CrossValidator(estimator = lrs, estimatorParamMaps = param_grid, evaluator = f1_evaluation, numFolds = 3)
model_lr_cv = cv_lr.fit(train)
results_lr_cv = model_lr_cv.transform(val)

print('Logistic Regression Cross-Val metrics:')
evaluator = MulticlassClassificationEvaluator(predictionCol='prediction')
print('Accuracy: {}'.format(evaluator.evaluate(results_lr_cv, {evaluator.metricName: "accuracy"})))
print('F-1 Score:{}'.format(evaluator.evaluate(results_lr_cv, {evaluator.metricName: "f1"})))
print('Recall Score:{}'.format(evaluator.evaluate(results_lr_cv, {evaluator.metricName: "weightedRecall"})))
model_rf = rf.fit(train)
results_rf = model_rf.transform(val)

print('Random Forest metrics:')
evaluator = MulticlassClassificationEvaluator(predictionCol='prediction')
print('Accuracy: {}'.format(evaluator.evaluate(results_rf, {evaluator.metricName: "accuracy"})))
print('F-1 Score:{}'.format(evaluator.evaluate(results_rf, {evaluator.metricName: "f1"})))
print('Recall Score:{}'.format(evaluator.evaluate(results_rf, {evaluator.metricName: "weightedRecall"})))
model_svc = svc.fit(train)
results_svc = model_svc.transform(val)

print('Linear SVC metrics:')
evaluator = MulticlassClassificationEvaluator(predictionCol='prediction')
print('Accuracy: {}'.format(evaluator.evaluate(results_svc, {evaluator.metricName: "accuracy"})))
print('F-1 Score:{}'.format(evaluator.evaluate(results_svc, {evaluator.metricName: "f1"})))
model_gb = gb.fit(train)
results_gb = model_gb.transform(val)

print('GB Tree metrics:')
evaluator = MulticlassClassificationEvaluator(predictionCol='prediction')
print('Accuracy: {}'.format(evaluator.evaluate(results_gb, {evaluator.metricName: "accuracy"})))
print('F-1 Score:{}'.format(evaluator.evaluate(results_gb, {evaluator.metricName: "f1"})))<jupyter_output>GB Tree metrics:
Accuracy: 0.7551020408163265
F-1 Score:0.7181433549029368<jupyter_text>Tuning Random Forest Model<jupyter_code>rf_tune = RandomForestClassifier()

param_grid_rf_tune = ParamGridBuilder().addGrid(rf_tune.maxDepth, [5, 10])\
                                   .addGrid(rf_tune.numTrees, [20, 30, 40, 50]).build()

f1_evaluation = MulticlassClassificationEvaluator(metricName = 'f1')

cv_rf_tune = CrossValidator(estimator = rf_tune, estimatorParamMaps = param_grid_rf_tune, evaluator = f1_evaluation, numFolds = 3)

model_rf_tune = cv_rf_tune.fit(train)
results_rf_tune = model_rf_tune.transform(val)

print('Tuned Random Forest metrics:')
evaluator = MulticlassClassificationEvaluator(predictionCol='prediction')
print('Accuracy: {}'.format(evaluator.evaluate(results_rf_tune, {evaluator.metricName: "accuracy"})))
print('F-1 Score:{}'.format(evaluator.evaluate(results_rf_tune, {evaluator.metricName: "f1"})))
model_rf_tune.bestModel.explainParams()
results_test = model_rf_tune.transform(test)<jupyter_output><empty_output><jupyter_text>Testing model performance for Final Model<jupyter_code>print('Tuned Random Forest metrics:')
evaluator = MulticlassClassificationEvaluator(predictionCol='prediction')
print('Accuracy: {}'.format(evaluator.evaluate(results_test, {evaluator.metricName: "accuracy"})))
print('F-1 Score:{}'.format(evaluator.evaluate(results_test, {evaluator.metricName: "f1"})))<jupyter_output>Tuned Random Forest metrics:
Accuracy: 0.7391304347826086
F-1 Score:0.6814774430966335<jupyter_text>Features in model: |total_songs|      total_length|thumbs_up|thumbs_down|user_lifetime| songs_per_session|friends_added|playlist_added|gender|total_artists|downgrade<jupyter_code>model_rf_tune.bestModel.featureImportances<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19021180341353075, 'fraction_numerical': 0.03166769483857701, 'mean_word_length': 5.164131812420786, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '196114', 'license': 'MIT', 'n_tokens_mistral': 3412, 'n_tokens_neox': 3251, 'n_words': 436, 'path': '6_Sparkify_Capstone/Sparkify Capstone.ipynb', 'repo_name': 'Sakchhi/Udacity_DS_Nanodegree', 'size': '9726'}