<jupyter_start><jupyter_text>Metadata and Full User Data  Random Forest Classifier<jupyter_code>%%time
rfc  = RandomForestClassifier(numTrees=200, maxDepth=10, 
                              labelCol="label", seed=42, 
                              maxMemoryInMB= 8192, 
                              featureSubsetStrategy = 'auto',
                              minInstancesPerNode = 20,
                              # minInfoGain = ?,
                              subsamplingRate = 0.6,
                              maxBins = 10)
rfc_model = rfc.fit(rmuf_train)
%%time
rfc_model_preds = rfc_model.transform(rmuf_test)
rfc_model_preds.show(2)
%%time
evaluator = MulticlassClassificationEvaluator(predictionCol="prediction")
print 'f1 score:', evaluator.evaluate(rfc_model_preds)
print 'Weighted Precision score:', evaluator.evaluate(rfc_model_preds, {evaluator.metricName: "weightedPrecision"})
print 'Weighted Recall score:', evaluator.evaluate(rfc_model_preds, {evaluator.metricName: "weightedRecall"})
print 'Accuracy:', evaluator.evaluate(rfc_model_preds, {evaluator.metricName: "accuracy"})<jupyter_output>f1 score: 0.264468191464
Weighted Precision score: 0.336493997634
Weighted Recall score: 0.345180319496
Accuracy: 0.345180319496
CPU times: user 123 ms, sys: 33.2 ms, total: 156 ms
Wall time: 10min 14s<jupyter_text>Random Forest Regressor<jupyter_code>%%time
rfr  = RandomForestRegressor(numTrees=200, maxDepth=10, 
                              labelCol="label", seed=42, 
                              maxMemoryInMB= 8192, 
                              featureSubsetStrategy = 'auto',
                              minInstancesPerNode = 20,
                              # minInfoGain = ?,
                              subsamplingRate = 0.6)
rfr_model = rfr.fit(rmuf_train)
rfr_model_preds = rfr_model.transform(rmuf_test)
rfr_model_preds.show(2)
evaluator = RegressionEvaluator(predictionCol="prediction")
print 'RMSE score:', evaluator.evaluate(rfr_model_preds)
print 'R-squared:', evaluator.evaluate(rfr_model_preds, {evaluator.metricName: "r2"})
print 'Mean Absolute Error:', evaluator.evaluate(rfr_model_preds, {evaluator.metricName: "mae"})<jupyter_output>RMSE score: 1.68841502653
R-squared: 0.355981716853
Mean Absolute Error: 1.29749801383<jupyter_text>Gradient Boosted Tree Regressor Depth 30 (Long run time)<jupyter_code>%%time
gbtr  = GBTRegressor(maxDepth=30, labelCol="label", seed=42, subsamplingRate=0.7, stepSize = 0.1, maxMemoryInMB= 8192)
gbtr_model = gbtr.fit(rmuf_train)
%%time
gbtr_model_preds = gbtr_model.transform(rmuf_test)
gbtr_model_preds.show(2)
%%time
evaluator = RegressionEvaluator(predictionCol="prediction")
print 'RMSE score:', evaluator.evaluate(gbtr_model_preds)
print 'R-squared:', evaluator.evaluate(gbtr_model_preds, {evaluator.metricName: "r2"})
print 'Mean Absolute Error:', evaluator.evaluate(gbtr_model_preds, {evaluator.metricName: "mae"})<jupyter_output><empty_output><jupyter_text>Depth 10<jupyter_code>%%time
gbtr_2  = GBTRegressor(maxDepth=10, labelCol="label", seed=42, stepSize = 0.1, maxMemoryInMB= 2048)
gbtr_model_2 = gbtr_2.fit(rmuf_train)
# gbtr_model.featureImportances
%%time
gbtr_model_preds_2 = gbtr_model_2.transform(rmuf_test)
gbtr_model_preds_2.show(2)
%%time
evaluator = RegressionEvaluator(predictionCol="prediction")
print 'RMSE score:', evaluator.evaluate(gbtr_model_preds_2)
print 'R-squared:', evaluator.evaluate(gbtr_model_preds_2, {evaluator.metricName: "r2"})
print 'Mean Absolute Error:', evaluator.evaluate(gbtr_model_preds_2, {evaluator.metricName: "mae"})
# Save and load model
gbtr_model_2.save(data_path + 'GBTRegD10Model_20m')
# sameModel = GBTRegressionModel.load(sc, 'hdfs:///user/andrew/GBTRegD10Model_20m')
sameModel = GBTRegressionModel.load(data_path + 'GBTRegD10Model_20m')
%%time
sameModel_preds_2 = sameModel.transform(rmuf_test)
sameModel_preds_2.show(2)
%%time
evaluator = RegressionEvaluator(predictionCol="prediction")
print 'RMSE score:', evaluator.evaluate(sameModel_preds_2)
print 'R-squared:', evaluator.evaluate(sameModel_preds_2, {evaluator.metricName: "r2"})
print 'Mean Absolute Error:', evaluator.evaluate(sameModel_preds_2, {evaluator.metricName: "mae"})
# %%time
# from pyspark.ml.tuning import ParamGridBuilder, CrossValidator
# gbtr = GBTRegressor(seed = 42, maxMemoryInMB= 2048)
# grid = ParamGridBuilder() \
#         .addGrid(gbtr.maxDepth, [10, 20]) \
#         .addGrid(gbtr.subsamplingRate, [0.6, 0.7]) \
#         .addGrid(gbtr.stepSize, [0.01, 0.05, 0.1]) \
#         .build()
# evaluator = RegressionEvaluator(predictionCol="prediction", metricName="mae")
# cv = CrossValidator(estimator=gbtr, estimatorParamMaps=grid, evaluator=evaluator, seed = 42)
# cv_model = cv.fit(rmuf_sdf_train_2)
# evaluator.evaluate(cv_model.transform(rmuf_sdf_train_2))
# evaluator.evaluate(cv_model.transform(rmuf_sdf_test_2))
# params = [{p.name: v for p, v in m.items()} for m in cv_model.getEstimatorParamMaps()]
# [ps.update({cv_model.getEvaluator().getMetricName(): metric}) for ps, metric in zip(params, cv_model.avgMetrics)]
# params_df = pd.DataFrame(params)
# print 'Best Parameters:'
# params_df.iloc[np.where(np.argmin(params_df.metric))]
sc.stop()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1209382810997885, 'fraction_numerical': 0.043260911363199386, 'mean_word_length': 4.691466083150985, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '187815', 'license': 'Apache-2.0', 'n_tokens_mistral': 2038, 'n_tokens_neox': 1854, 'n_words': 306, 'path': 'Model_Building/Content_Based_Filtering_GBT_and_Random_Forest_20m.ipynb', 'repo_name': 'Dell-AI/Recommendation-System-in-Spark', 'size': '5201'}