<jupyter_start><jupyter_text>python文本挖掘  1 数据读入<jupyter_code>import pandas as pd
df = pd.read_csv('train_content.csv',encoding='utf-8')
df<jupyter_output><empty_output><jupyter_text>2 分词<jupyter_code>import jieba
import pandas as pd
df.head(10)
#df.drop('num',axis=1)
#df = df.drop_duplicates(['Data','num']).reset_index()
df = df[['Data']]
df
df.dropna(inplace=True)
 df
titles = df['Data']#提取需分词的内容
#titles.head(10)

#按词性分词
import jieba.posseg as pseg
titles1=[]
for title in titles:
    title1=pseg.cut(title)
    title2=[[word.word,word.flag]for word in title1]
    titles1.append(title2)
#titles1
print(titles1[1])
#按词性抽取
titles2=[[word[0] for word in title if word[1]=='n' 
         or word[1]=='v' or word[1]=='a' or word[1]=='d' or word[1]=='nr' 
         or word[1]=='nrfg' or word[1]=='ns' or word[1]=='nt' or word[1]=='nz' 
         or word[1]=='vn' or word[1]=='vd' or word[1]=='an' or word[1]=='ad' 
         or word[1]=='l']for title in titles1]
#按词性抽取
titles2=[[word[0] for word in title if word[1]=='n'] for title in titles1]
print(titles2[1])
#删除一个字的词
titles3 =[[word for word in text if len(word)>1]for text in titles2]
print(titles3[1])
titles3<jupyter_output><empty_output><jupyter_text>3 计算词频<jupyter_code>#合并成一个文档
text = []
for word in titles3:
    text.extend(word)
#统计频数
from collections import defaultdict
word_dict = defaultdict(int) 
for word in text:
    word_dict[word] +=1
word_dict.items()
#从字典中提取词条和词频
terms = []
freq = []
for item in word_dict.items():
    terms.append(item[0])
    freq.append(item[1])
wordcount = pd.DataFrame({'terms':terms,'count':freq})#写入数据框
wordcount1 = wordcount.sort_values(by='count',ascending=False)#按词频降序排列
wordcount1

wordcount1.to_csv('allWordsCount.csv')
#以空格分割
titles4 = [' '.join(word) for word in titles3]
df['titlecut'] = titles4
df<jupyter_output><empty_output><jupyter_text>TF-IDF<jupyter_code>#定义用于tfidf的分词器
def tokench(text):
    tokens = [word for word in text.split(' ')]#数据以空格隔开
    after_tokens = []
    for token in tokens:
        if len(token)>1:
            after_tokens.append(token)
    return after_tokens
#构建tfidf矩阵
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=200000,
                                 tokenizer=tokench,use_idf=True)

tfidf_matrix = tfidf_vectorizer.fit_transform(df['titlecut']) #计算任何两个文档的相似度
terms = tfidf_vectorizer.get_feature_names()#获取词（特征）名
terms[:10]
weight = tfidf_matrix.todense().tolist()#tfidf权重<jupyter_output><empty_output><jupyter_text>文本相似度<jupyter_code>#计算距离
from sklearn.metrics.pairwise import cosine_similarity
sim = cosine_similarity(tfidf_matrix)
dist = 1.0 - cosine_similarity(tfidf_matrix)#根据两个文档的词条的TF-IDF值来计算
sim[5,111]
[s for s in sim]<jupyter_output><empty_output><jupyter_text>K-means 聚类<jupyter_code>#利用scikit-learn中的Kmeans模型进行聚类
from sklearn.cluster import KMeans

num_clusters = 9

km = KMeans(n_clusters=num_clusters)

km.fit(tfidf_matrix)

clusters = km.labels_.tolist()
clusters[:10]#前10个文档的类
df['cluster'] = clusters
df.tail()
df.to_csv('clusters.csv')<jupyter_output><empty_output><jupyter_text>谱系聚类<jupyter_code>import pandas as pd
# df = pd.read_csv('train_content.csv', error_bad_lines=False,header=None)
df = pd.read_csv('train_content.csv',encoding='utf-8')
df = df[['Data']]
df.dropna(inplace=True)
df<jupyter_output><empty_output><jupyter_text>nltk分词<jupyter_code>import nltk
texts = []
for text in df['Data']:
    tokens = [token.lower() for token in nltk.word_tokenize(text)]
    tagged = nltk.pos_tag(tokens)
    tagged1 = [[word[0],word[1]] for word in tagged]
    texts.append(tagged1)
texts[1]
##按词性提取
texts1 = [[word[0] for word in tagged if word[1]=='JJ' or word[1]=='JJR' or word[1]=='JJS'
        or word[1]=='FW' or word[1]=='NN' or word[1]=='NNS' or word[1]=='NNP'
        or word[1]=='NNPS' or word[1]=='RB' or word[1]=='RBR' or word[1]=='RBS'
        or word[1]=='RP' or word[1]=='VB' or word[1]=='VBD' or word[1]=='VBG'
        or word[1]=='VBN' or word[1]=='VBP' or word[1]=='VBZ'] for tagged in texts]
texts1[1]<jupyter_output><empty_output><jupyter_text>过滤stopwords<jupyter_code>from nltk.corpus import stopwords
stopwords = stopwords.words('english')
texts2 = [[word for word in text if word not in stopwords] for text in texts1]
#以空格分割
texts3 = [' '.join(text) for text in texts2]
df['descut'] = texts3
df
#定义用于tfidf的分词器
def tokench(text):
    tokens = [word for word in text.split(' ')]#数据以空格隔开
    after_tokens = []
    for token in tokens:
        if len(token)>1:
            after_tokens.append(token)
    return after_tokens
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=200000,
                                 tokenizer=tokench,use_idf=True)

tfidf_matrix = tfidf_vectorizer.fit_transform(df['descut'])
#计算距离矩阵
from sklearn.metrics.pairwise import cosine_similarity
dist = 1 - cosine_similarity(tfidf_matrix)<jupyter_output><empty_output><jupyter_text>利用scipy做谱系聚类<jupyter_code># from scipy.cluster.hierarchy import ward, dendrogram
# import matplotlib.pyplot as plt
# %matplotlib inline

# linkage_matrix = ward(dist) #通过ward法构建矩阵

# fig, ax = plt.subplots(figsize=(15,20))
# ax = dendrogram(linkage_matrix, orientation='left')#画图的

# plt.tick_params(axis= 'x',          
#                 which='both',      
#                 bottom='off',      
#                 top='off',         
#                 labelbottom='off')

# plt.tight_layout() #紧凑布局

# #保存图像
# plt.savefig('ward_clusters.png', dpi=300)<jupyter_output><empty_output><jupyter_text>主题建模<jupyter_code>from gensim import corpora, models, similarities
texts = titles3
dictionary = corpora.Dictionary(texts) #构造词典
dictionary.filter_extremes(no_below=1, no_above=0.8)#去高频词

corpus = [dictionary.doc2bow(text) for text in texts] #构造语料库
[s for s in corpus]
#训练一个LDA模型
%time lda = models.LdaModel(corpus, num_topics=9, id2word=dictionary, update_every=5, alpha=0.1, eta=0.1)
print(lda[corpus[0]])#打印第一个文档的主题建模结果<jupyter_output>[(0, 0.57892376), (1, 0.052633613), (2, 0.052632548), (3, 0.052632283), (4, 0.052633125), (5, 0.052642047), (6, 0.05263526), (7, 0.052633625), (8, 0.052633736)]<jupyter_text>**预测文档对应的主题**<jupyter_code>doctop = [ s for s in lda[corpus]]
doctop = pd.DataFrame(doctop)
doctop
doctop.to_csv('DocumentTopic.csv')<jupyter_output><empty_output><jupyter_text>**主题对应的关键词**<jupyter_code>topicwords = lda.print_topics(num_words=20)
topicwords<jupyter_output><empty_output><jupyter_text>**模型可视化**<jupyter_code>import pyLDAvis
pyLDAvis.enable_notebook(local=True)
import pyLDAvis.gensim
pyLDAvis.gensim.prepare(lda, corpus, dictionary)<jupyter_output><empty_output><jupyter_text>情感分析  中文情感分析<jupyter_code>from snownlp import SnowNLP
df = pd.read_csv('train_content.csv',encoding='utf-8')
df = df[['Data']]
df.dropna(inplace=True)
sentiments = [SnowNLP(text).sentiments for text in df['Data']]
sentiments[1]
df['sentiment'] = sentiments
df.to_csv('shfbsent.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15173203549957057, 'fraction_numerical': 0.03478385342112797, 'mean_word_length': 5.666984732824427, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '228956', 'license': 'MIT', 'n_tokens_mistral': 3114, 'n_tokens_neox': 3068, 'n_words': 492, 'path': 'Test1/Code/Code.ipynb', 'repo_name': 'zlzzq/--python', 'size': '6986'}