<jupyter_start><jupyter_text>Loading Data<jupyter_code>feature_culumns = ["_c" + str(i+1) for i in range(784)]
df_training = (spark
               .read
               .options(header = False, inferSchema = True)
               .csv(f"file://{path}/mnist-data/mnist_train.csv"))

from pyspark.ml.feature import VectorAssembler

vectorizer = VectorAssembler(inputCols=feature_culumns, outputCol="features")
training = (vectorizer
            .transform(df_training)
            .select("_c0", "features")
            .toDF("label", "features")
            .repartition(15)
            .cache())
df_testing = (spark
              .read
              .options(header = False, inferSchema = True)
              .csv(f"file://{path}/mnist-data/mnist_test.csv"))

testing = (vectorizer
           .transform(df_testing)
           .select("_c0", "features")
           .toDF("label", "features")
           .cache())<jupyter_output><empty_output><jupyter_text>Multilayer Perceptron Classifier<jupyter_code>%%time
from pyspark.ml.classification import MultilayerPerceptronClassifier

layers = [28*28, 300, 10]

mpc = MultilayerPerceptronClassifier(maxIter=30, layers=layers)

model = mpc.fit(training)
from pyspark.sql.functions import expr
result = model.transform(testing).withColumn("matched", expr("label == prediction"))
result.show(3)
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
evaluator = MulticlassClassificationEvaluator(labelCol="label", predictionCol="prediction", metricName="accuracy")
evaluator.evaluate(result)
wrong_df = result.filter("matched = false")
images = wrong_df.take(36)
import matplotlib.pyplot as plt

fig, _ = plt.subplots(6, 6, figsize = (20, 20))
for i, ax in enumerate(fig.axes):
    r = images[i]
    label = r.label
    prediction = int(r.prediction)
    features = r.features
    ax.imshow(features.toArray().reshape(28, 28), cmap = "Greys")
    ax.set_title(f"True: {str(label)} / Pred: {str(prediction)}")

plt.show()
spark.stop()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13647642679900746, 'fraction_numerical': 0.01588089330024814, 'mean_word_length': 3.732394366197183, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '113452', 'license': 'MIT', 'n_tokens_mistral': 690, 'n_tokens_neox': 661, 'n_words': 119, 'path': 'V9/2_MNIST_MultilayerPerceptronClassifier.ipynb', 'repo_name': 'fred1234/BDLC_FS22', 'size': '2015'}