<jupyter_start><jupyter_text>数据清洗和绘图观察数据 ![image.png](attachment:image.png)该demo实现目标:- 绘制文章词云- 利用word2vec查找具有高相关性的词语，并绘图显示<jupyter_code>!pip install pyecharts==0.5.11 && pip install baidu-aip
import pandas as pd
import numpy as np
import warnings
import re
from sklearn.manifold import TSNE
import jieba
import jieba.analyse
import matplotlib.pyplot as plt
from pyecharts import WordCloud
from collections import Counter
import multiprocessing
from gensim.models import Word2Vec
from aip import AipNlp
from gensim import corpora, models
from gensim.models.ldamodel import LdaModel

# 忽略警告
warnings.filterwarnings('ignore')
# 查看数据的基本格式
df = pd.read_csv('./data/JianShuJiaoYou-All-Data.csv',encoding='utf-8')
print(df.columns)
# df.head(5)<jupyter_output>Index(['Page', 'Num', 'Author', 'Author_Url', 'Title', 'Time', 'Read',
       'Comment', 'Like', 'Abstract', 'Artical_Url', 'Image_Url',
       'Artical_Content'],
      dtype='object')<jupyter_text>提取文章中200个关键词绘制词云<jupyter_code>content = df['Artical_Content'].tolist()
# print(len(content),'\n',content[-1])
# 载入停用词
# quoting : int or csv.QUOTE_* instance, default 0
# 控制csv中的引号常量。可选 QUOTE_MINIMAL (0), QUOTE_ALL (1), QUOTE_NONNUMERIC (2) or QUOTE_NONE (3)
 
stopwords = pd.read_csv('./data/chinesestopword.txt',sep='\n',encoding='utf-8',names=['stopwords'],header=None,quoting=3)
# 默认输出5个
stopwords.head()
# 结巴分词
pattern = re.compile('\d+')

# 存放词语和词频
wordsCounter = Counter()

for line in content:
    segs = jieba.lcut(line)
    for seg in segs:
        if len(seg)>1 and seg != '\r\n' and re.search(pattern,seg)==None:
            wordsCounter[seg] += 1

print(wordsCounter.most_common(100))

# 将Counter的键提取出来做list
segment = list(wordsCounter)

# 将分好的词列表转化为词典
words = pd.DataFrame({'segment':segment})

# 剔除停用词
words = words[~words['segment'].isin(stopwords['stopwords'])]
# 绘制词云
def counter2list(_counter):
    wordslist,nums = [],[]
    for item in _counter:
        wordslist.append(item[0])
        nums.append(item[1])
    return wordslist,nums

outputFile = './result/文章关键词词云图.html'

# 这个关键词抽取方法不唯一
wordslist,nums = counter2list(wordsCounter.most_common(1000))

cloud = WordCloud('文章关键词词云', width=1200, height=600, title_pos='center')
cloud.add(
    ' ',
    wordslist,nums,
    shape='circle',
    background_color='white',
    max_words=1000 
)

cloud.render(outputFile)<jupyter_output><empty_output><jupyter_text>word2vec模型计算词的相似度<jupyter_code># 基于TF-IDF算法的关键词抽取
contentStr = ''.join(content)
# keyWordsList = jieba.analyse.extract_tags(contentStr,topK=200,allowPOS=('ns','n'))
# textrank1 = ' '.join(keyWordsList)
# textrank1
# 基于TextRank算法进行关键词抽取
textrank2 = ' '.join(jieba.analyse.textrank(contentStr,topK=500,allowPOS=('ns','n')))
textrank2
# 利用word2vec找出关联词语，将语料整理成嵌套列表的形式
corpus = []

# 将停用词dataFrame转化为列表
stopwordsList = stopwords['stopwords'].tolist()
# print(stopwordsList[:10])
for line in content:
    segs = jieba.lcut(line)
    segs = filter(lambda x:len(x)>1, segs)
    segs = filter(lambda x:re.search(pattern,x)==None, segs)
    segs = filter(lambda x:x not in stopwordsList, segs)
    
    corpus.append(list(segs))
# 训练模型
model = Word2Vec(corpus,min_count=20,sg=0,workers=multiprocessing.cpu_count())
# 查询和简书相关性比较高的词语
model.wv.most_similar(['简书'],topn=15)<jupyter_output><empty_output><jupyter_text>t-SNE 可视化高维数据t-SNE是目前最为流行的一种高维数据降维的算法<jupyter_code># 调用百度云NLP进行词向量表示
APP_ID = '11617353'
API_KEY = '<KEY>'
SECRET_KEY = '<KEY>'
client = AipNlp(APP_ID, API_KEY, SECRET_KEY)
# 获取textrank2中Top200词语的词向量
textrankList = textrank2.split(' ')
# 整理格式 原始格式：('简书', 0.9286492277441794) 后者是特征权重
words_list = []
word_vectors = []

for word in textrankList:
    try:
        data = client.wordEmbedding(word)
#         print(data)
        word_vector = data['vec']
        words_list.append(data['word'])
        word_vectors.append(word_vector)
    except:
        print('No words:{}'.format(word))
    
word_vectors = np.array(word_vectors)
# print(words_list)
def plotTsne2D(word_vectors,words_list):
    tsne = TSNE(n_components=2,random_state=0,n_iter=10000,perplexity=20)
    # 在控制台输出过程中，默认小数会以科学计数法的形式输出，若不需要加上下面这句
    np.set_printoptions(suppress=True)
    T = tsne.fit_transform(word_vectors)
    labels = words_list
    
    plt.figure(figsize=(20,18))
    plt.scatter(T[:,0],T[:,1],c='blue',edgecolors='k')
    
    for label,x,y in zip(labels,T[:,0],T[:,1]):
        plt.annotate(label,xy=(x+1,y+1),xytext=(0,0),textcoords='offset points')

# 解决中文显示问题
plt.rcParams['font.sans-serif'] = ['Kai']
plt.rcParams['axes.unicode_minus'] = False

plotTsne2D(word_vectors,words_list)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1485278080697928, 'fraction_numerical': 0.025736095965103598, 'mean_word_length': 5.589080459770115, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '657840', 'license': 'MIT', 'n_tokens_mistral': 2056, 'n_tokens_neox': 2088, 'n_words': 275, 'path': 'Day10-NLP/word2vec(avg)_DNN.ipynb', 'repo_name': 'shaiic/ai-lab', 'size': '4631'}