<jupyter_start><jupyter_text>ml lib<jupyter_code>from pyspark.ml.clustering import KMeans
from pyspark.ml.evaluation import ClusteringEvaluator
from pyspark.ml.feature import VectorAssembler
def kmeans_mllib(filename, k):
    flat_data = loadData(filename).map(lambda x: x[1])  
    
    features = ["c1", "c2", "c3", "c4"]
    dataset = spark.createDataFrame(flat_data, features)

     
    
    transformed_dataset = VectorAssembler(inputCols=features, outputCol="features").transform(dataset) 
    
    
    kmeans = KMeans(k=k, seed=1) 
    model = kmeans.fit(transformed_dataset.select('features'))
    
    return model.transform(transformed_dataset) 
    
kmeans_mllib = kmeans_mllib('iris_input.data', 4)

 
kmeans_mllib.groupBy('prediction').count().show()

points.map(lambda x: (x[1][0], x[0]) ).groupByKey().mapValues(lambda x: len(list(x))).collect()<jupyter_output>+----------+-----+
|prediction|count|
+----------+-----+
|         1|   23|
|         3|   30|
|         2|   47|
|         0|   50|
+----------+-----+
{'fraction_non_alphanumeric': 0.1836734693877551, 'fraction_numerical': 0.021379980563654033, 'mean_word_length': 3.12, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '172096', 'license': 'Apache-2.0', 'n_tokens_mistral': 389, 'n_tokens_neox': 366, 'n_words': 58, 'path': 'projet_data_analitics/ipynb_data/spark_kmean_vanilla_impl.ipynb', 'repo_name': 'Zouhairhajji/Data_science', 'size': '1029'}