<jupyter_start><jupyter_text>Data Loading<jupyter_code>#import neccessary libraries
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.feature import StringIndexer
from pyspark.ml.feature import VectorIndexer
#load training data
trainData = spark.read.format("csv") \
  .option("header", "true") \
  .option("inferSchema", "true") \
  .option("sep", ",") \
  .load("/FileStore/tables/9fruit_csv/9fruitsTrainDF.csv")

trainData.cache() # Cache data for faster reuse

#load testing data
testData = spark.read.format("csv") \
  .option("header", "true") \
  .option("inferSchema", "true") \
  .option("sep", ",") \
  .load("/FileStore/tables/9fruit_csv/9fruitsTestDF.csv")

testData.cache() # Cache data for faster reuse
column2features = trainData.columns
column2features.remove("label")
#create feature column
assembler = VectorAssembler().setInputCols(column2features).setOutputCol("features")
trainData_with_features = assembler.transform(trainData)
testData_with_features = assembler.transform(testData)

# trainData_with_features.show(1)
# display(trainData_with_features.limit(10))
#filter rows with null features
trainData_with_features = trainData_with_features.filter("features is not null")
testData_with_features = testData_with_features.filter("features is not null")
# index label
# add metadata to label column
# fit on whole dataset so that to include all labels in index
labelIndexer = StringIndexer().setInputCol("label").setOutputCol("indexedLabel").setHandleInvalid("skip").fit(trainData_with_features)
# index feature
# identify categorical features, and index them.
# set maxCategories as 20 so features with > 20 distinct values are treated as continuous
featureIndexer = VectorIndexer().setInputCol("features").setOutputCol("indexedFeatures").setMaxCategories(20).setHandleInvalid("skip").fit(trainData_with_features)<jupyter_output><empty_output><jupyter_text>Multi-layer Perceptron Classifier<jupyter_code>#import neccessary libraries
from pyspark.ml.classification import MultilayerPerceptronClassifier
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.ml.feature import IndexToString
from pyspark.ml import Pipeline
# last 9 is the predicted value output
# 9 equals to 9 value
layers = [784, 50, 16, 9]

trainer = MultilayerPerceptronClassifier().setLayers(layers).setLabelCol("indexedLabel").setFeaturesCol("indexedFeatures").setBlockSize(128).setSeed(123).setMaxIter(200)

labelConverter = IndexToString().setInputCol("prediction").setOutputCol("predictedLabel").setLabels(labelIndexer.labels)

pipeline = Pipeline().setStages([labelIndexer, featureIndexer, trainer, labelConverter])

model = pipeline.fit(trainData_with_features)

predictions = model.transform(testData_with_features)
display(predictions.select(['label','prediction']).limit(5))
#accuracy
evaluator = MulticlassClassificationEvaluator(predictionCol="prediction",labelCol = "label",metricName = "accuracy")
evaluator.evaluate(predictions)
#precision
evaluator = MulticlassClassificationEvaluator(predictionCol="prediction",labelCol = "label",metricName = "weightedPrecision")
evaluator.evaluate(predictions)
#recall
evaluator = MulticlassClassificationEvaluator(predictionCol="prediction",labelCol = "label",metricName = "weightedRecall")
evaluator.evaluate(predictions)
#F1-score
evaluator = MulticlassClassificationEvaluator(predictionCol="prediction",labelCol = "label",metricName = "f1")
evaluator.evaluate(predictions)<jupyter_output><empty_output><jupyter_text>Random Forest Classifier<jupyter_code>#import neccessary libraries
from pyspark.ml.classification import RandomForestClassifier
from pyspark.ml import Pipeline
from pyspark.ml.tuning import ParamGridBuilder, CrossValidator
# Create a RandomForestClassifier
rf = RandomForestClassifier()

# Set ParamGridBuilder 
grid_rf = ParamGridBuilder().baseOn({rf.labelCol: 'label'}).baseOn([rf.predictionCol, 'prediction']).baseOn([rf.featuresCol,'features']).baseOn([rf.seed,123]).addGrid(rf.maxDepth, [5,10]).addGrid(rf.numTrees, [10,20]).build()

evaluator = MulticlassClassificationEvaluator(predictionCol="prediction",labelCol = "label",metricName = "accuracy")


cv_rf = CrossValidator(estimator=rf, estimatorParamMaps=grid_rf, evaluator=evaluator)

# Create a Pipeline
rfPipeline = Pipeline()

# Set the stages of the Pipeline
rfPipeline.setStages([assembler, cv_rf])

# Let's train on the entire dataset to see what we get
rfModel = rfPipeline.fit(trainData)
resultsDF_rf = rfModel.transform(testData).select('label','prediction')
display(resultsDF_rf.limit(10))
#accuracy
evaluator = MulticlassClassificationEvaluator(predictionCol="prediction",labelCol = "label",metricName = "accuracy")
evaluator.evaluate(resultsDF_rf)
#precision
evaluator = MulticlassClassificationEvaluator(predictionCol="prediction",labelCol = "label",metricName = "weightedPrecision")
evaluator.evaluate(resultsDF_rf)
#recall
evaluator = MulticlassClassificationEvaluator(predictionCol="prediction",labelCol = "label",metricName = "weightedRecall")
evaluator.evaluate(resultsDF_rf)
#F1-score
evaluator = MulticlassClassificationEvaluator(predictionCol="prediction",labelCol = "label",metricName = "f1")
evaluator.evaluate(resultsDF_rf)<jupyter_output><empty_output><jupyter_text>Multinomial Logistic Regression<jupyter_code>#import neccessary libraries
from pyspark.ml.classification import LogisticRegression
# Create a Multinomial Logistic Regression Model
mlr = LogisticRegression(family = "multinomial")

# Set ParamGridBuilder 
grid_mlr = ParamGridBuilder().baseOn({mlr.labelCol: 'label'}).baseOn([mlr.predictionCol, 'prediction']).baseOn([mlr.featuresCol,'features']).addGrid(mlr.maxIter, [50,100]).addGrid(mlr.regParam, [1.0,2.0]).build()

evaluator = MulticlassClassificationEvaluator(predictionCol="prediction",labelCol = "label",metricName = "accuracy")

cv_mlr = CrossValidator(estimator=mlr, estimatorParamMaps=grid_mlr, evaluator=evaluator)

# Create a Pipeline
mlrPipeline = Pipeline()

# Set the stages of the Pipeline
mlrPipeline.setStages([assembler, cv_mlr])

# Let's train on the entire dataset to see what we get
mlrModel = mlrPipeline.fit(trainData)
resultsDF_mlr = mlrModel.transform(testData).select('label','prediction')
display(resultsDF_mlr.limit(10))
#accuracy
evaluator = MulticlassClassificationEvaluator(predictionCol="prediction",labelCol = "label",metricName = "accuracy")
evaluator.evaluate(resultsDF_mlr)
#precision
evaluator = MulticlassClassificationEvaluator(predictionCol="prediction",labelCol = "label",metricName = "weightedPrecision")
evaluator.evaluate(resultsDF_mlr)
#recall
evaluator = MulticlassClassificationEvaluator(predictionCol="prediction",labelCol = "label",metricName = "weightedRecall")
evaluator.evaluate(resultsDF_mlr)
#F1-score
evaluator = MulticlassClassificationEvaluator(predictionCol="prediction",labelCol = "label",metricName = "f1")
evaluator.evaluate(resultsDF_mlr)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1155010814708003, 'fraction_numerical': 0.009516943042537852, 'mean_word_length': 9.525037936267072, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '548111', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 2213, 'n_tokens_neox': 2159, 'n_words': 376, 'path': 'project.ipynb', 'repo_name': 'marylfo/FruitsDoodleClassifiers', 'size': '6935'}