<jupyter_start><jupyter_text>**Machine Learning and Advanced Analytics**<jupyter_code>staticDataFrame.printSchema()
from pyspark.sql.functions import date_format, col
preppedDataFrame = staticDataFrame\
    .na.fill(0)\
    .withColumn("day_of_week", date_format(col("InvoiceDate"),"EEEE"))\
    .coalesce(5)
#Split th data into training and test sets
trainDataFrame = preppedDataFrame\
    .where("InvoiceDate < '2011-07-01'")
testDataFrame = preppedDataFrame\
    .where("InvoiceDate >= '2011-07-01'")
trainDataFrame.count()
testDataFrame.count()
from pyspark.ml.feature import StringIndexer
indexer = StringIndexer()\
    .setInputCol("day_of_week")\
    .setOutputCol("day_of_week_index")
# fix the days issue
from pyspark.ml.feature import OneHotEncoder
encoder = OneHotEncoder()\
    .setInputCol("day_of_week_index")\
    .setOutputCol("day_of_week_encoded")
from pyspark.ml.feature import VectorAssembler
vectorAssembler = VectorAssembler()\
    .setInputCols(["UnitPrice","Quantity", "day_of_week_encoded"])\
    .setOutputCol("features")
from pyspark.ml import Pipeline
transformationPipeline = Pipeline()\
    .setStages([indexer, encoder, vectorAssembler])
fittedPipeline = transformationPipeline.fit(trainDataFrame)
transformedTraining.cache()
from pyspark.ml.clustering import KMeans
kmeans = KMeans()\
    .setK(20)\
    .setSeed(1)
kmModel = kmeans.fit(transformedTraining)
kmModel.computeCost(transformedTraining)
transforedTest = fittedPipeline.transform(testDataFrame)

kmModel.computeCost(transformedTest)
**Lower-Level APIs**
from pyspark.sql import Row

spark.sparkContext.parallelize([Row(1), Row(2), Row(3)]).toDF()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12402636309167166, 'fraction_numerical': 0.014379868184541641, 'mean_word_length': 6.324561403508772, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '629900', 'license': 'Apache-2.0', 'n_tokens_mistral': 579, 'n_tokens_neox': 550, 'n_words': 98, 'path': "I.Gentle_Overview_of_Big_Data_and_Spark/3.A_Tour_Of_Spark's_Toolset/Structured_Streaming.ipynb", 'repo_name': 'NikolayVaklinov10/Spark_The_Definitive_Guide', 'size': '1669'}