<jupyter_start><jupyter_text>k-means示例开始，以20newsgroup数据集为例<jupyter_code>from sklearn import datasets
from sklearn.feature_extraction.text import TfidfVectorizer
all_data = datasets.fetch_20newsgroups(subset='all')
# 简单起见，只用部分新闻组，用categories指定
groups = ['comp.graphics', 'comp.os.ms-windows.misc','comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 
'comp.windows.x', 'sci.space']
train_data = datasets.fetch_20newsgroups(subset='train', categories=groups)
test_data = datasets.fetch_20newsgroups(subset='test', categories=groups)
print(len(test_data.filenames))
print(len(train_data.filenames))
# 忽略不合法字符  decode_error='ignore'
vectorizer_km = TfidfVectorizer(min_df=10, max_df=.5, stop_words='english', decode_error='ignore')
vectorized = vectorizer_km.fit_transform(train_data.data)
k = 50
from sklearn.cluster import KMeans
# verbose 打印出训练的信息
km = KMeans(n_clusters=k, init='k-means++', n_init=1, verbose=5)
km.fit(vectorized)
# 用训练的模型开始预测
new_post_km = '''
Disk drive problems. Hi, I have a problem with my hard disk.
After 1 year it is working only sporadically now.
I tried to format it, but now it doesn't boot any more.
Any ideas? Thanks.
'''
new_post_km_vec = vectorizer_km.transform([new_post_km])
new_post_label = km.predict(new_post_km_vec)[0]
# 找到训练集中与new_post_km属于同一簇的索引
# nonzero返回非0的索引
similar_indices = (km.labels_ == new_post_label).nonzero()[0]
print("查看索引：")
print(list(similar_indices))
print("维度相同：")
print(vectorized[0].shape)
print(new_post_km_vec.shape)
similar = []    # 相似文章
for i in similar_indices:
    dis = sp.linalg.norm((new_post_km_vec - vectorized[i]).toarray())
    similar.append((dis, train_data.data[i]))
sorted(similar)
print(similar[-1])
test = vectorizer_km.transform(test_data.data)
km.score(test)<jupyter_output><empty_output><jupyter_text>k-means示例<jupyter_code>import matplotlib.pyplot as plt
import seaborn as sns;sns.set()
from sklearn.datasets.samples_generator import make_blobs
x, y = make_blobs(n_samples=300, centers=4,cluster_std=0.6, random_state=0)
plt.scatter(x[:,0], x[:,1], s=30)
plt.show()
cluster_num = 4
km1 = KMeans(n_clusters=cluster_num)
km1.fit(x)
y_predict = km1.predict(x)
print(y_predict)
plt.scatter(x[:,0], x[:,1], c=y_predict, s=30, cmap='rainbow')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14146772767462423, 'fraction_numerical': 0.01768346595932803, 'mean_word_length': 7.83984375, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '421687', 'license': 'MIT', 'n_tokens_mistral': 947, 'n_tokens_neox': 937, 'n_words': 150, 'path': 'building machine learning systems with python/k-means.ipynb', 'repo_name': 'GalileoSama/MachineLearning', 'size': '2262'}