<jupyter_start><jupyter_text>Augrund den Erkenntnissen aus dem Notebook Dataunderstanding wird der Feature Vektor auf 7 Merkmale reduziert<jupyter_code>featureCols = df.columns.copy()
featureCols.remove("class")
featureCols.remove("cap-shape")
featureCols.remove("cap-surface")
featureCols.remove("cap-color")
featureCols.remove("odor")
featureCols.remove("bruises")
featureCols.remove("gill-attachment")
featureCols.remove("gill-spacing")
featureCols.remove("gill-size")
featureCols.remove("gill-color")
featureCols.remove("stalk-shape")
featureCols.remove("stalk-root")
featureCols.remove("stalk-surface-above-ring")
featureCols.remove("stalk-surface-below-ring")
featureCols.remove("stalk-color-above-ring")
featureCols.remove("stalk-color-below-ring")
featureCols.remove("veil-type")
featureCols.remove("veil-color")
featureCols.remove("ring-number")
featureCols.remove("ring-type")
featureCols.remove("spore-print-color")
featureCols.remove("population")
featureCols.remove("habitat")

#featureCols = featureCols + ["cap_shapeNUM","cap_surfaceNUM", "cap_colorNUM", "bruisesNUM", "odorNUM", "gill_attachmentNUM", "gill_spacingNUM", "gill_sizeNUM", "gill_colorNUM", "stalk_shapeNUM", "stalk_rootNUM", "stalk_surface_above_ringNUM", "stalk_surface_below_ringNUM", "stalk_color_above_ringNUM", "stalk_color_below_ringNUM", "veil_colorNUM", "ring_numberNUM", "ring_typeNUM", "spore_print_colorNUM", "populationNUM", "habitatNUM"]
featureCols = featureCols + ["odorNUM","gill_sizeNUM", "gill_colorNUM", "stalk_rootNUM", "ring_typeNUM", "spore_print_colorNUM", "populationNUM"]

print(featureCols)
labeledData = labelIndexer.transform(df)

#labeledDataNUM = cap_shapeIndexer.transform(cap_surfaceIndexer.transform(cap_colorIndexer.transform(bruisesIndexer.transform(odorIndexer.transform(gill_attachmentIndexer.transform(gill_spacingIndexer.transform(gill_sizeIndexer.transform(gill_colorIndexer.transform(stalk_shapeIndexer.transform(stalk_rootIndexer.transform(stalk_surface_above_ringIndexer.transform(stalk_surface_below_ringIndexer.transform(stalk_color_above_ringIndexer.transform(stalk_color_below_ringIndexer.transform(veil_colorIndexer.transform(veil_typeIndexer.transform(ring_numberIndexer.transform(ring_typeIndexer.transform(spore_print_colorIndexer.transform(populationIndexer.transform(habitatIndexer.transform(labeledData))))))))))))))))))))))
labeledDataNUM = odorIndexer.transform(gill_sizeIndexer.transform(gill_colorIndexer.transform(stalk_rootIndexer.transform(ring_typeIndexer.transform(spore_print_colorIndexer.transform(populationIndexer.transform(labeledData)))))))
assembler =  VectorAssembler(outputCol="features", inputCols=featureCols)
labeledPointData = assembler.transform(labeledDataNUM)

splits = labeledPointData.randomSplit([0.9, 0.1 ], 12345)
training = splits[0]
test = splits[1]
## SVM Klassifizierung
evaluator = BinaryClassificationEvaluator(labelCol="label",rawPredictionCol="prediction", metricName="areaUnderROC")
lsvc = LinearSVC(labelCol="label",aggregationDepth=2, featuresCol="features") 
SVMModel = lsvc.fit(training)
predictions = SVMModel.transform(test)
## Test Error und Confusion Matrix ausgeben
accuracy = evaluator.evaluate(predictions)
print("Test Error", (1.0 - accuracy))
predictionAndLabels = predictions.select("prediction", "label").rdd.map(lambda p: [p[0], float(p[1])])
metrics = MulticlassMetrics(predictionAndLabels)
confusion = metrics.confusionMatrix()
print("Confusion matrix: \n" , confusion)
spark.stop()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1322101656196459, 'fraction_numerical': 0.0048543689320388345, 'mean_word_length': 12.630350194552529, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '74123', 'license': 'Apache-2.0', 'n_tokens_mistral': 1188, 'n_tokens_neox': 1124, 'n_words': 128, 'path': 'files/exercises/.ipynb_checkpoints/mushroom_classification_SVM-checkpoint.ipynb', 'repo_name': 'erikschmitt/DataScienceW20', 'size': '3502'}