<jupyter_start><jupyter_text>+ 对quest和answer做预处理，求词向量和句向量<jupyter_code>import cut_sentence as cs

all_sentence = []
for idx in range(len(qa_corpus)):
    q = qa_corpus['question'][idx]
    if isinstance(q,str):
        question = cs.segment(q,'arr')
        all_sentence.append(question)
    a = qa_corpus['answer'][idx]
    if isinstance(a,str):
        answer = cs.segment(a,'arr')
        all_sentence.append(answer)

# 用fastText求所有的词向量
from gensim.models import FastText

model = FastText(all_sentence,size=4, window=3, min_count=1, iter=10,min_n = 3 , max_n = 6,word_ngrams = 0)
# 做句向量
X_sentences= []
new_sentences = []
for idx in range(len(all_sentence)):
    sentence = all_sentence[idx]
    if len(sentence)>0:
        vec = sum([model.wv[word] for word in sentence])/len(sentence)
        X_sentences.append(vec)
        new_sentences.append(sentence)<jupyter_output><empty_output><jupyter_text>意图识别  意图识别的目标是将问题分类+ 先对语料库中的问题进行聚类<jupyter_code># K-means的K能否用随机行走的方式来得到？
# 随机从一个点出发，寻找离这个点最近距离的点，然后移动到最近点，接着从这个点继续移动到下一个最近点，这条路径记为L；
# 寻找一个合适的K，使得K个团的L总和最小，并且每个K团之间的距离最大（尽可能分开）
# 使用K-mean聚类
from sklearn.cluster import KMeans

n_clusters = 5
kmean = KMeans(n_clusters=n_clusters)
kmean.fit(X_sentences)
print("kmean: k={}, cost={}".format(n_clusters, int(kmean.score(X_sentences))))
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn.decomposition import PCA

labels = kmean.labels_
centers = kmean.cluster_centers_
X_plot = np.array(X_sentences)
pca = PCA(n_components=2)   #降到2维
pca.fit(X_plot)                  #训练
newX=pca.fit_transform(X_plot)   #降维后的数据

fig = plt.figure()
# ax = fig.gca(projection='3d')
ax = fig.gca()
ax.scatter(newX[:, 0], newX[:, 1], c=labels, cmap=plt.cm.Spectral)
plt.show()
# 测试一下，输入句子属于哪一类
input = '你好'
x_quest = cs.segment(input,'arr')
x_vec = sum([model.wv[word] for word in x_quest])/len(x_quest)

kmean.predict([x_vec])
joblib.dump(kmean , 'km.pkl')<jupyter_output><empty_output><jupyter_text>语义相似度判断<jupyter_code>kmean = joblib.load('km.pkl')
print(kmean.labels_)

def get_class(model,sentence):
    return model.predict([vectorize(sentence)])
from sklearn.neighbors import KDTree

class DataSpace():
    def __init__(self,classes_data):
        '''
        构建KD树
        '''
        points = len(classes_data)
        leaf = points/2-1
        self.tree = KDTree(classes_data,leaf)
        
    def __vectorize__(self,sentence):
        x_quest = cs.segment(sentence,'arr')
        return [sum([model.wv[word] for word in x_quest])/len(x_quest)]
    
    def similarity(self,sentence,topk=10):
        '''
        从KD树中找到最相似的k个数据
        '''
        v = self.__vectorize__(sentence)
        dist, ind = self.tree.query(v,k=topk)
        return (dist,ind)
data0 = DataSpace(np.array(X_sentences)[kmean.labels_==0])
sentence1 = np.array(new_sentences)[kmean.labels_==0]
data1 = DataSpace(np.array(X_sentences)[kmean.labels_==1])
sentence2 = np.array(new_sentences)[kmean.labels_==1]
data2 = DataSpace(np.array(X_sentences)[kmean.labels_==2])
sentence3 = np.array(new_sentences)[kmean.labels_==2]
data3 = DataSpace(np.array(X_sentences)[kmean.labels_==3])
sentence4 = np.array(new_sentences)[kmean.labels_==3]
data4 = DataSpace(np.array(X_sentences)[kmean.labels_==4])
sentence5 = np.array(new_sentences)[kmean.labels_==4]
_,ind = data2.similarity('你好')
sentence3[ind]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14298427766241473, 'fraction_numerical': 0.012755858795609611, 'mean_word_length': 4.874564459930314, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '195067', 'license': 'Apache-2.0', 'n_tokens_mistral': 1465, 'n_tokens_neox': 1497, 'n_words': 192, 'path': 'test/corpus.ipynb', 'repo_name': 'webbery/lovechat', 'size': '3371'}