<jupyter_start><jupyter_text>LDA topic model* reference: https://blog.csdn.net/TiffanyRabbit/article/details/76445909  1. load data<jupyter_code>#加载数据
from sklearn.datasets import fetch_20newsgroups
n_samples=2000
dataset = fetch_20newsgroups(shuffle=True, random_state=1,
                             remove=('headers', 'footers', 'quotes'))

data_samples = dataset.data[:n_samples] #截取需要的量，n_samples=2000
len(data_samples)
data_samples[:3]<jupyter_output><empty_output><jupyter_text>2. preprocess* lower casing* remove stopwords* tokenization* stemmer* ...  3. CountVectorizer<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer

tf_vectorizer = CountVectorizer(max_df=0.95, min_df=2,
                                max_features=200,
                                stop_words='english')
tf = tf_vectorizer.fit_transform(data_samples)
tf.shape<jupyter_output><empty_output><jupyter_text>4. LDA & result<jupyter_code>from sklearn.decomposition import LatentDirichletAllocation
n_topic = 10
lda = LatentDirichletAllocation(n_topic, 
                                max_iter=50,
                                learning_method='batch')
lda.fit(tf)                           
# 输出10个主题下，影响较高的5个词语


def print_top_words(model, feature_names, n_top_words):
    #打印每个主题下权重较高的term
    for topic_idx, topic in enumerate(model.components_):
        print("Topic #%d:" % topic_idx)
        print(" ".join([feature_names[i]  for i in topic.argsort()[:-n_top_words - 1:-1]]))
        
    #打印主题-词语分布矩阵
    print(model.components_)

n_top_words=5
tf_feature_names = tf_vectorizer.get_feature_names()
print_top_words(lda, tf_feature_names, n_top_words)


lda.components_.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10262390670553936, 'fraction_numerical': 0.02507288629737609, 'mean_word_length': 3.6504065040650406, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '242095', 'license': 'MIT', 'n_tokens_mistral': 633, 'n_tokens_neox': 618, 'n_words': 86, 'path': 'sklearn/lda_topic_model_by_sklearn.ipynb', 'repo_name': 'ybdesire/machinelearning', 'size': '1715'}