<jupyter_start><jupyter_text>Random Forest Regression - The example below is using **boston dataset** for preciting the **median home prices** for occupied homes.- The example is stricitly used for **demonstrating** the uagae of `EnsembleTreeExplainTransformer`.<jupyter_code>boston_csv = '/work/tests/data/regression/dataset_boston.csv'
boston_df: DataFrame = spark.read.option("header", True).option("inferSchema", True).csv(boston_csv)
label_column = 'medv'
features_column = f"features_{label_column}"
prediction_column = f"prediction_{label_column}"

contrib_column = f"prediction_{label_column}_contrib"
contrib_column_sum = f"{contrib_column}_sum"
contrib_column_intercept = f"{contrib_column}_intercept"

features_importance_view = f"features_importance_{label_column}_view"
predictions_view = f"predictions_{label_column}_view"
categorical_columns=[]
continuous_columns = [x for x in boston_df.columns if x not in ['id', label_column]]
stages = get_ensemble_pipeline_stages([], continuous_columns + categorical_columns, label_column, ensemble_type="rf",classification=False)
import uuid
pipeline = Pipeline(stages=stages)
pipeline_model: PipelineModel = pipeline.fit(boston_df)
rf_model = pipeline_model.stages[-1]
string_id = uuid.uuid4()
rf_model_path: str = f"/tmp/{string_id}"
rf_model.write().save(rf_model_path)
prediction_df = pipeline_model.transform(boston_df)
prediction_df.createOrReplaceTempView(predictions_view)
features_importance_df = get_feature_importance(spark, rf_model, prediction_df, features_column)
features_importance_df.createOrReplaceTempView(features_importance_view)
explain_stages = get_ensemble_explain_stages(predictions_view, features_importance_view, label_column, rf_model_path,ensemble_type="rf",classification=False)
explain_pipeline = Pipeline(stages=explain_stages)
explain_df = explain_pipeline.fit(prediction_df).transform(prediction_df)
feature_index_meta = features_importance_df.select("Feature_Index", "Feature").orderBy("Feature_Index").rdd.collectAsMap()
expressions = []
for key,value in feature_index_meta.items():
    expressions.append(f"AVG({contrib_column}[{key}]) as {value}")

contributions = explain_df.selectExpr(expressions).collect()[0].asDict()
plot_contrib_df = pd.DataFrame.from_dict(contributions,orient='index',columns=['Contributions'])
plot_contrib_df.head()
final_plot_contrib_df = plot_contrib_df.iloc[plot_contrib_df['Contributions'].abs().argsort()]
final_plot_contrib_df.plot(kind='barh', figsize=(9, 7))
plt.title('Feature Contributions')
plt.axvline(x=0, color='.5')
plt.subplots_adjust(left=.3)<jupyter_output><empty_output><jupyter_text>Random Forest Classification - The example below is using **prima india diabetes dataset** for preciting the **diabetes outcome** for patient.- The example is stricitly used for **demonstrating** the uagae of `GLMExplainTransformer`.<jupyter_code>prima_indian_diabetes_csv = '/work/tests/data/classification/dataset_prima_indian_diabetes.csv'
prima_indian_diabetes_df = spark.read.option("header", True).option("inferSchema", True).csv(
    prima_indian_diabetes_csv)
label_column = 'outcome'
features_column = f"features_{label_column}"
prediction_column = f"prediction_{label_column}"

contrib_column = f"prediction_{label_column}_contrib"
contrib_column_sum = f"{contrib_column}_sum"
contrib_column_intercept = f"{contrib_column}_intercept"

features_importance_view = f"features_importance_{label_column}_view"
predictions_view = f"predictions_{label_column}_view"
categorical_columns=[]
continuous_columns = [x for x in prima_indian_diabetes_df.columns if x not in ['id', label_column]]
stages = get_ensemble_pipeline_stages([], continuous_columns + categorical_columns, label_column, ensemble_type="rf",classification=True)
import uuid
pipeline = Pipeline(stages=stages)
pipeline_model: PipelineModel = pipeline.fit(prima_indian_diabetes_df)
rf_model = pipeline_model.stages[-1]
string_id = uuid.uuid4()
rf_model_path: str = f"/tmp/{string_id}"
rf_model.write().save(rf_model_path)
prediction_df = pipeline_model.transform(prima_indian_diabetes_df)
prediction_df.createOrReplaceTempView(predictions_view)
features_importance_df = get_feature_importance(spark, rf_model, prediction_df, features_column)
features_importance_df.createOrReplaceTempView(features_importance_view)
explain_stages = get_ensemble_explain_stages(predictions_view, features_importance_view, label_column, rf_model_path,ensemble_type="rf",classification=True)
explain_pipeline = Pipeline(stages=explain_stages)
explain_df = explain_pipeline.fit(prediction_df).transform(prediction_df)
feature_index_meta = features_importance_df.select("Feature_Index", "Feature").orderBy("Feature_Index").rdd.collectAsMap()
expressions = []
for key,value in feature_index_meta.items():
    expressions.append(f"AVG({contrib_column}[{key}]) as {value}")

contributions = explain_df.selectExpr(expressions).collect()[0].asDict()
plot_contrib_df = pd.DataFrame.from_dict(contributions,orient='index',columns=['Contributions'])
plot_contrib_df.head()
final_plot_contrib_df = plot_contrib_df.iloc[plot_contrib_df['Contributions'].abs().argsort()]
final_plot_contrib_df.plot(kind='barh', figsize=(9, 7))
plt.title('Feature Contributions')
plt.axvline(x=0, color='.5')
plt.subplots_adjust(left=.3)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11446351122006411, 'fraction_numerical': 0.003017160098057703, 'mean_word_length': 11.193103448275862, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '237619', 'license': 'Apache-2.0', 'n_tokens_mistral': 1770, 'n_tokens_neox': 1737, 'n_words': 259, 'path': 'examples/notebooks/spark/pyspark_random_forest_explain.ipynb', 'repo_name': 'clarifyhealth/transparency', 'size': '5303'}