<jupyter_start><jupyter_text>1. Features : The input features used for training of the model2. Affairs : The true labels for the records3. rawPrediction : a measure of confidence in each possible label (where larger = more confident).4. probability : The Probability is the conditional probability for each class5. prediction : The Prediction is the result of finding the statistical mode of the rawPrediction<jupyter_code>from pyspark.ml.evaluation import MulticlassClassificationEvaluator
accuracy=MulticlassClassificationEvaluator(labelCol='affairs',metricName='accuracy').evaluate(predictions)
accuracy
accuracy=MulticlassClassificationEvaluator(labelCol='affairs',metricName='f1').evaluate(predictions)
accuracy
precision=MulticlassClassificationEvaluator(labelCol='affairs',metricName='weightedPrecision').evaluate(predictions)
precision
recall=MulticlassClassificationEvaluator(labelCol='affairs',metricName='weightedRecall').evaluate(predictions)
recall
from pyspark.ml.evaluation import BinaryClassificationEvaluator
auc=BinaryClassificationEvaluator(labelCol='affairs').evaluate(predictions)
auc
## Random Forest
from pyspark.ml.classification import RandomForestClassifier
rf_classifier=RandomForestClassifier(labelCol='affairs').fit(train_df)
rf_predictions=rf_classifier.transform(test_df)
rf_predictions.select(['probability','affairs','prediction']).show(10,False)
rf_accuracy=MulticlassClassificationEvaluator(labelCol='affairs',metricName='accuracy').evaluate(rf_predictions)
rf_accuracy
rf_precision=MulticlassClassificationEvaluator(labelCol='affairs',metricName='weightedPrecision').evaluate(rf_predictions)
rf_precision
#AUC

rf_auc=BinaryClassificationEvaluator(labelCol='affairs').evaluate(rf_predictions)
rf_auc
# Feature importance
rf_classifier.featureImportances
df.schema["features"].metadata["ml_attr"]["attrs"]
#increased number of trees
rf_classifier=RandomForestClassifier(labelCol='affairs',numTrees=500).fit(train_df)
rf_predictions=rf_classifier.transform(test_df)
rf_accuracy=MulticlassClassificationEvaluator(labelCol='affairs',metricName='accuracy').evaluate(rf_predictions)
rf_accuracy
rf_precision=MulticlassClassificationEvaluator(labelCol='affairs',metricName='weightedPrecision').evaluate(rf_predictions)
rf_precision
df.schema["features"].metadata["ml_attr"]["attrs"]
rf_classifier.featureImportances<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09772535804549284, 'fraction_numerical': 0.004633529907329402, 'mean_word_length': 15.156462585034014, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '415107', 'license': 'Apache-2.0', 'n_tokens_mistral': 713, 'n_tokens_neox': 683, 'n_words': 104, 'path': 'a01_PySpark/e01_Resources/Machine-Learning-with-Pyspark/Decision Trees.ipynb', 'repo_name': 'mindis/Big_Data_Analysis', 'size': '2374'}