<jupyter_start><jupyter_text>Boston Housing Classification Logistic Regression<jupyter_code>from pyspark.sql.types import BooleanType
from pyspark.ml.feature import StringIndexer, VectorAssembler
from pyspark.mllib.classification import LogisticRegressionWithLBFGS
from pyspark.sql.session import SparkSession
from pyspark.sql.functions import expr
from pyspark.ml.evaluation import BinaryClassificationEvaluator
from pyspark.mllib.regression import LabeledPoint
from pyspark.mllib.linalg import DenseVector
from pyspark.mllib.evaluation import MulticlassMetrics
inputFile = "../data/Boston_Housing_Data.csv"<jupyter_output><empty_output><jupyter_text>Spark session creation<jupyter_code>spark = (SparkSession
       .builder
       .appName("BostonHousingClass")
       .getOrCreate())<jupyter_output><empty_output><jupyter_text>DataFrame creation using an ifered Schema<jupyter_code>df = spark.read.option("header", "true") \
       .option("inferSchema", "true") \
       .option("delimiter", ";") \
       .csv(inputFile) \
       .withColumn("CATBOOL", expr("CAT").cast(BooleanType()))
print(df.printSchema())
featureCols = df.columns.copy()
featureCols.remove("MEDV")
featureCols.remove("CAT")
featureCols.remove("CATBOOL") 
print(featureCols)
assembler =  VectorAssembler(outputCol="features", inputCols=featureCols)
featureSet = assembler.transform(df)<jupyter_output><empty_output><jupyter_text>Build Labeled Point Feature Set<jupyter_code>featureSetLP = (featureSet.select(featureSet.CAT, featureSet.features)
      .rdd
      .map(lambda row: LabeledPoint(row.CAT, DenseVector(row.features))))<jupyter_output><empty_output><jupyter_text>Prepare training and test data.<jupyter_code>splits = featureSetLP.randomSplit([0.9, 0.1 ], 12345)
training = splits[0]
test = splits[1]<jupyter_output><empty_output><jupyter_text>Logistic regression Train the model<jupyter_code>#TODO Optimize the parameters of LR
modelLRLB = LogisticRegressionWithLBFGS.train(training, numClasses=2)<jupyter_output><empty_output><jupyter_text>Test the model<jupyter_code>predictionAndLabels = test.map(lambda x : [float(modelLRLB.predict(x.features )), float(x.label) ])
print (predictionAndLabels.take(20))<jupyter_output><empty_output><jupyter_text>evaluate the result<jupyter_code>metrics =  MulticlassMetrics(predictionAndLabels)
print("Test Error LRLBFG =" , (1.0 - metrics.accuracy))
spark.stop()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11469979296066253, 'fraction_numerical': 0.006625258799171843, 'mean_word_length': 7.659498207885305, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '462906', 'license': 'Apache-2.0', 'n_tokens_mistral': 837, 'n_tokens_neox': 806, 'n_words': 125, 'path': 'files/exercises/.ipynb_checkpoints/BostonHousingClassificationLogRegLBFG-checkpoint.ipynb', 'repo_name': 'erikschmitt/DataScienceW20', 'size': '2415'}