<jupyter_start><jupyter_text>K-Mean<jupyter_code>'''
KMeans clustering - engineer new feature based on cluster results
Use K=3 to fit the assembled features
'''
from pyspark.ml.clustering import KMeans

feature_label_assembler = VectorAssembler(inputCols=df1.columns[1:], outputCol="assemebled")
cluster_df = feature_label_assembler.transform(df1)

kmeans = KMeans(k=3, featuresCol='assemebled', 
                predictionCol='clusterClassPrediction', distanceMeasure='euclidean',)

model = kmeans.fit(cluster_df)
transformed = model.transform(cluster_df).select("sig_id", "clusterClassPrediction")

df2 = df2.join(transformed, on=['sig_id'], how='inner')
df2.select("clusterClassPrediction").show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12602739726027398, 'fraction_numerical': 0.010958904109589041, 'mean_word_length': 7.914634146341464, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '378388', 'license': 'MIT', 'n_tokens_mistral': 250, 'n_tokens_neox': 240, 'n_words': 41, 'path': 'misc/attampt-k-mean-clustering-feat-engineering.ipynb', 'repo_name': 'shasha3493/Mechanism-of-Action-Prediction', 'size': '730'}