<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Load data2&nbsp;&nbsp;Look at data types2.1&nbsp;&nbsp;categorical values3&nbsp;&nbsp;Modelling4&nbsp;&nbsp;Model Predictions5&nbsp;&nbsp;Model evaluation<jupyter_code>import numpy as np
import pandas as pd
import pyspark
from pyspark import SparkConf, SparkContext, SQLContext
from pyspark.sql import SparkSession
from pyspark.sql.functions import udf # @udf("integer") def myfunc(x,y): return x - y
from pyspark.sql import functions as F # stddev format_number date_format, dayofyear, when
from pyspark.sql.types import StructField, StringType, IntegerType, StructType

print([(x.__name__,x.__version__) for x in [np, pd, pyspark]])

spark = pyspark.sql.SparkSession.builder.appName('tree').getOrCreate()
sc = spark.sparkContext
sqlContext = SQLContext(sc) # spark_df = sqlContext.createDataFrame(pandas_df)
sc.setLogLevel("INFO")
from pyspark.ml.feature import StringIndexer, VectorIndexer,OneHotEncoder
from pyspark.ml.feature import VectorAssembler
from pyspark.ml import Pipeline

from pyspark.ml.regression import LinearRegression
from pyspark.ml.regression import DecisionTreeRegressor
from pyspark.ml.regression import RandomForestRegressor
from pyspark.ml.regression import GBTRegressor

from pyspark.ml.classification import LogisticRegression
from pyspark.ml.classification import DecisionTreeClassifier
from pyspark.ml.classification import RandomForestClassifier
from pyspark.ml.classification import GBTClassifier

from pyspark.ml.evaluation import BinaryClassificationEvaluator
from pyspark.ml.evaluation import MulticlassClassificationEvaluator<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>!head -2 ../data/College.csv
data = spark.read.csv('../data/College.csv', header=True, inferSchema=True)
print(data.count())
print(data.printSchema())
data.limit(5).toPandas().T<jupyter_output>777
root
 |-- School: string (nullable = true)
 |-- Private: string (nullable = true)
 |-- Apps: integer (nullable = true)
 |-- Accept: integer (nullable = true)
 |-- Enroll: integer (nullable = true)
 |-- Top10perc: integer (nullable = true)
 |-- Top25perc: integer (nullable = true)
 |-- F_Undergrad: integer (nullable = true)
 |-- P_Undergrad: integer (nullable = true)
 |-- Outstate: integer (nullable = true)
 |-- Room_Board: integer (nullable = true)
 |-- Books: integer (nullable = true)
 |-- Personal: integer (nullable = true)
 |-- PhD: integer (nullable = true)
 |-- Terminal: integer (nullable = true)
 |-- S_F_Ratio: double (nullable = true)
 |-- perc_alumni: integer (nullable = true)
 |-- Expend: integer (nullable = true)
 |-- Grad_Rate: integer (nullable = true)

None<jupyter_text>Look at data types<jupyter_code>data.dtypes
numeric_features = [t[0] for t in data.dtypes if t[1] == 'int']
data.select(numeric_features).describe().toPandas().transpose()
print(data.columns)
input_cols = [ 'Apps', 'Accept', 'Enroll', 'Top10perc', 'Top25perc',
              'F_Undergrad', 'P_Undergrad', 'Outstate', 'Room_Board',
              'Books', 'Personal', 'PhD', 'Terminal', 'S_F_Ratio',
              'perc_alumni', 'Expend', 'Grad_Rate']

assembler = VectorAssembler(inputCols=input_cols, outputCol='features')
output = assembler.transform(data)<jupyter_output><empty_output><jupyter_text>categorical values<jupyter_code>categorical_features = [t[0] for t in data.dtypes if t[1] == 'string']
categorical_features
data.select('Private').distinct().show()
data.select('School').distinct().show()
indexer = StringIndexer(inputCol='Private', outputCol='PrivateIndex')
output_fixed = indexer.fit(output).transform(output)
output_fixed.printSchema()
final_data = output_fixed.select('features','PrivateIndex')
train_data, test_data = final_data.randomSplit([0.7, 0.3],seed=100)<jupyter_output><empty_output><jupyter_text>Modelling<jupyter_code>from pyspark.ml.classification import DecisionTreeClassifier,GBTClassifier,RandomForestClassifier
dtc = DecisionTreeClassifier(labelCol='PrivateIndex', featuresCol='features')
rfc = RandomForestClassifier(labelCol='PrivateIndex', featuresCol='features',numTrees=200)
gbt = GBTClassifier(labelCol='PrivateIndex', featuresCol='features')
dtc_model = dtc.fit(train_data)
rfc_model = rfc.fit(train_data)
gbt_model = gbt.fit(train_data)<jupyter_output><empty_output><jupyter_text>Model Predictions<jupyter_code>dtc_preds = dtc_model.transform(test_data)
rfc_preds = rfc_model.transform(test_data)
gbt_preds = gbt_model.transform(test_data)
dtc_preds.show(2)<jupyter_output>+--------------------+------------+-------------+-----------+----------+
|            features|PrivateIndex|rawPrediction|probability|prediction|
+--------------------+------------+-------------+-----------+----------+
|[152.0,128.0,75.0...|         0.0|  [241.0,0.0]|  [1.0,0.0]|       0.0|
|[191.0,165.0,63.0...|         0.0|  [241.0,0.0]|  [1.0,0.0]|       0.0|
+--------------------+------------+-------------+-----------+----------+
only showing top 2 rows<jupyter_text>Model evaluation<jupyter_code>from pyspark.ml.evaluation import MulticlassClassificationEvaluator
acc_evaluator = MulticlassClassificationEvaluator(labelCol="PrivateIndex",
                                                  predictionCol="prediction",
                                                  metricName="accuracy")
dtc_acc = acc_evaluator.evaluate(dtc_preds)
rfc_acc = acc_evaluator.evaluate(rfc_preds)
gbt_acc = acc_evaluator.evaluate(gbt_preds)
output_df = pd.DataFrame({'DT': dtc_acc, 'RF': rfc_acc, 'GBT': gbt_acc},
                         index=['accuracy']).mul(100).round(2)
output_df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16208496965369512, 'fraction_numerical': 0.01606569082470546, 'mean_word_length': 4.5420375865479725, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '346753', 'license': 'Apache-2.0', 'n_tokens_mistral': 1877, 'n_tokens_neox': 1825, 'n_words': 339, 'path': 'Section13_Decision_Trees_Random_Forest/tree_methods_code_along.ipynb', 'repo_name': 'bhishanpdl/Spark_and_Python_for_Big_Data_with_PySpark_Udemy', 'size': '5602'}